included rabbitmq as broker

This commit is contained in:
2026-11-10 07:01:17 +03:00
parent 0a70358829
commit 2b9fb3f375
49 changed files with 3072 additions and 287 deletions
+22 -4
View File
@@ -1,5 +1,5 @@
# Скопируйте в .env для локального окружения; .env не коммитится.
# Настройки приложения и Compose; параметры будущей очереди пока не используются.
# Настройки приложения, Compose и очереди lab5.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
@@ -72,10 +72,28 @@ MINIO_ROOT_USER_FILE=/run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE=/run/secrets/minio_root_password
MINIO_BROWSER=off
# Lab5: будущие параметры очереди/worker, пока без реализации.
# Lab5: отдельный RabbitMQ broker и ограниченный worker.
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
# Брокер пока не выбран; файл ещё не создаётся и не монтируется.
BROKER_URL_FILE=/run/secrets/broker_url
BROKER_HOST=broker
BROKER_PORT=5672
BROKER_PASSWORD_FILE=/run/secrets/broker_password
BROKER_QUEUE=attachment-jobs
BROKER_USER=messenger
BROKER_VHOST=/
JOB_LEASE_SECONDS=90
JOB_RETRY_SECONDS=5
PUBLISHER_POLL_SECONDS=1
# Качество вариантов и лимиты одного процесса обработки
IMAGE_JPEG_QUALITY=85
IMAGE_WEBP_QUALITY=80
IMAGE_WEBP_METHOD=4
IMAGE_MEMORY_MIB=768
IMAGE_CPU_SECONDS=20
# Retry cap and retained failed sources for explicit replay
JOB_RETRY_MAX_SECONDS=120
FAILED_STAGING_RETENTION_SECONDS=604800
+29 -5
View File
@@ -43,7 +43,8 @@ endef
rolling-update failover-all test-lab3-grade4
help:
@echo "Lab4: init | test-grade4 | test-s3 | load-uploads | cleanup-dry-run | cleanup-apply"
@echo "Lab5: init | worker-stop | worker-start | jobs-logs | cleanup-dry-run | cleanup-apply"
@echo "Lab5: test-grade4 | test-jobs; legacy lab4: test-s3 | load-uploads"
@echo "Проверки: check | validate | test | load | test-persistence ACTION_SCRIPT=scripts/restart.sh"
@echo " test-failover ACTION_SCRIPT=scripts/stop-one.sh"
@echo "Основной: up | down | restart | restore | ps | logs | migrate | instances"
@@ -65,10 +66,13 @@ init:
test-grade4:
$(MAKE) check validate
$(MAKE) test-stand-up
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover
$(TEST_ENV) $(PYTHON) -m scripts.check_s3 --base-url "$(TEST_BASE_URL)" --with-outage --with-faults
$(TEST_ENV) $(PYTHON) -m scripts.upload_load --base-url "$(TEST_BASE_URL)" --concurrency 10
$(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)"
$(MAKE) test-docker
$(MAKE) test-stand-smoke
$(MAKE) test-stand-idempotency
$(MAKE) test-stand-persistence
$(MAKE) test-stand-failover
$(TEST_ENV) $(PYTHON) -m scripts.check_outbox_recovery --base-url "$(TEST_BASE_URL)"
$(TEST_ENV) $(PYTHON) -m scripts.check_lab5_jobs --base-url "$(TEST_BASE_URL)"
test-s3:
$(PYTHON) -m scripts.check_s3 --base-url "$(BASE_URL)" --with-outage --with-faults
@@ -226,3 +230,23 @@ expiry-demo: test-stand-up
failure-demo: test-stand-up
$(TEST_ENV) BASE_URL=$(TEST_BASE_URL) scripts/failure_demo.sh
# --- Очередь lab5 ---
.PHONY: worker-stop worker-start jobs-logs
worker-stop:
$(COMPOSE) stop worker
worker-start:
$(COMPOSE) start worker
jobs-logs:
$(COMPOSE) logs -f publisher worker broker
# Уже работающий изолированный стенд; сам цель его не поднимает.
.PHONY: test-outbox-recovery
test-outbox-recovery:
$(PYTHON) -m scripts.check_outbox_recovery --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: test-jobs
test-jobs:
$(PYTHON) -m scripts.check_lab5_jobs --base-url "$(BASE_URL)"
+203 -53
View File
@@ -1,75 +1,225 @@
# Отчёт — лабораторная 5
- Автор, группа: Григорян Илья Мурадович, БПИ2303 (из отчёта lab4).
- Целевая оценка: 5; исследовательский трек пока не выбран.
- Этап: перенос базы lab4 и заготовки с docstring, без реализации lab5.
- Дата переноса: 07.11.2026.
- Commit/PR lab5: заполнить после публикации.
- Использованные источники: README-файлы и contracts/ репозитория lab5,
исходники lab4. Использована помощь Codex. Новых зависимостей нет.
Автор: Григорян Илья Мурадович, БПИ2303.
Целевая оценка текущего этапа: **4**. Обновлено 10.11.2026.
Дальнейшая цель — 5; исследовательский трек ещё не выбран.
Использована помощь Codex при реализации, проверках и исправлениях.
Commit/PR: заполнить после публикации.
## Запуск из чистого clone
## Запуск
Нужны Python 3.12, pip, Docker с Compose. Подготовка и запуск:
Нужны Python 3.12, pip, Docker с Compose. Зависимости закреплены в requirements.txt.
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
make init
make up
make test
```
HTTP: http://localhost:18081. `make down` останавливает стенд с сохранением данных.
Тестовый стенд: `make test-stand-up`, остановка: `make test-stand-down`.
Намеренное удаление учебных данных: `make test-stand-destroy` для тестового стенда,
`docker compose --env-file .env down --volumes` для основного.
Версии зависимостей перенесены без обновления. Локальные secrets/.env генерируются заново, не копируются из lab4.
Стенд при переносе не запускался; воспроизводимость после переноса ещё не проверена.
`make init` создаёт отсутствующие .env и файлы secrets/, сохраняя существующие.
Все environment-параметры Compose подставляются из .env; образец — .env.example.
Пароли и S3 credentials читаются из файлов secrets, не включаются в задания.
API не получает credentials брокера. По умолчанию HTTP доступен на
http://localhost:18081. Broker, MinIO и основные PostgreSQL/Redis наружу не открыты.
## Архитектура и решения
При обновлении ранее запущенного стенда сначала остановить старые процессы
через `make down` без удаления volumes, затем выполнить `make up`.
Migrate применяет ревизии до a62c950e18b3 перед стартом приложения.
Миграции добавляют jobs, варианты, бюджет попыток и аудит replay. Готовые
вложения раннего варианта lab5 переобрабатываются для заполнения crop/WebP.
При переходе Redis Streams → RabbitMQ миграция повторно публикует queued jobs
из PostgreSQL; processing восстанавливаются по истечении lease. Старые процессы
обоих брокеров во время перехода должны быть остановлены.
Перенесены приложение FastAPI, PostgreSQL-миграции, общие Redis-сессии,
идемпотентность сообщений, приватный MinIO, две API-реплики за Nginx, собственные
тесты и скрипты эксплуатации lab4. Перенесена и незакоммиченная правка MinIO
bootstrap из исходного рабочего дерева. Исходный lab4 не изменён.
## Инфраструктура и устойчивый приём
Имена Compose-проектов, образы и порты отделены от lab4. Рабочие настройки —
.env.example; там же указаны будущие параметры очереди, пока не используемые.
Compose получает настройки environment из .env, включая пути S3, PostgreSQL
и MinIO к файлам секретов; секретные значения остаются в secrets/.
Пути *_FILE в Compose также задают точки монтирования соответствующих secrets;
MinIO bootstrap читает эти пути из окружения.
Публичные tests/, contracts/, lab.json и README.md lab5 сохранены.
| Компонент | Реализация | Данные |
| --- | --- | --- |
| API | две реплики FastAPI, Python 3.12.15 | PostgreSQL и Redis сессий |
| Балансировка | nginx 1.28.0 | единая HTTP-точка входа |
| База | PostgreSQL 16.10 | сообщения, metadata, jobs, outbox, audit |
| Сессии | Redis 7.4.2, AOF always | redisdata |
| Объекты | приватный MinIO | staging, originals, variants |
| Брокер | RabbitMQ 4.4.0 | durable очередь, rabbitmqdata |
| Publisher | отдельный процесс приложения, aio-pika 10.2.2 | PostgreSQL → RabbitMQ |
| Worker | отдельный процесс приложения | RabbitMQ → MinIO + PostgreSQL |
| Cleanup | отдельный процесс | незавершённые upload и terminal staging |
Будущая архитектура: API → durable staging + PostgreSQL job/outbox → publisher →
broker → worker → конечный MinIO + атомарная фиксация metadata в PostgreSQL.
Текущая реализация продолжает синхронно сохранять оригинал и возвращает 201 ready.
Заготовки lab5 не подключены и вызывают NotImplementedError. Реализация транзакционных границ и восстановления после отказов ещё предстоит.
Redis используется для сессий. Очередь заданий использует RabbitMQ через AMQP.
Очередь durable, сообщения persistent. Publisher ожидает подтверждение брокером
приёма сообщения (publisher confirm) и только затем фиксирует published_attempt.
При неизвестном результате публикации возможен дубль; намерение остаётся в БД.
Подтверждение worker (consumer ack) следует после устойчивого результата или
устойчиво сохранённого перехода retry/failed. При разрыве соединения RabbitMQ
возвращает неподтверждённые доставки в очередь.
## Доказательства критериев
До ответа `202 queued` API полностью принимает и проверяет оригинал, сохраняет
его в приватный staging и фиксирует attachment/job одной транзакцией PostgreSQL.
Job одновременно служит transactional outbox — устойчивым намерением публикации.
Worker и broker не входят в синхронный путь upload и readiness API. Остановка
worker не заставляет API выполнять resize. При компенсации неоднозначного COMMIT
API проверяет БД, чтобы не удалить оригинал уже принятого задания.
| Критерий README | Будущая команда/сценарий | Текущий результат | Доказательство |
| --- | --- | --- | --- |
| Публичный smoke lab5 | make test | Не запускался; 202/thumbnail ещё не реализованы | Нет |
| 3: сохранён минимум lab4 | Собственные тесты, persistence/failover | Перенесён, повторно не проверен | Не подтверждает lab5 |
| 3: durable приём и broker/worker | Worker stop → upload → restart → ready | Не реализовано | Нет |
| 3: состояния, original/thumbnail, retry, дубли | Smoke + собственные сценарии | Не реализовано | Нет |
| 4: crop/optimized, resource limits | Собственные проверки изображений/лимитов | Не реализовано | Нет |
| 4: outbox и восстановление COMMIT/publish | Fault injection | Не реализовано | Нет |
| 4: backoff/jitter, failed/replay | Сбой до исчерпания и явный replay | Не реализовано | Нет |
| 4: crash до ack, lease, concurrency, cleanup | Fault injection и lifecycle | Не реализовано | Нет |
| 5: законченный исследовательский трек | Методика после выбора трека | Трек не выбран | Нет |
Envelope соответствует contracts/jobs.schema.json: job_id, attachment_id,
pipeline_version=1, attempt, created_at, schema_version, type и необязательный
trace context. Bytes, имена объектов S3, URL и credentials туда не помещаются.
Доступ к вложениям проверяется по участию в чате; content до ready возвращает 409.
Аутентификация, пагинация, идемпотентность сообщений и ограничения lab4 сохранены.
## Эксперименты и отказы
## Изображения и ограничения
Пока не проводились. После выбора трека описать гипотезу, hardware/лимиты,
версии, размеры/число файлов, профиль нагрузки, concurrency/prefetch, длительность,
прогрев и повторы. Сохранить raw summary без секретов, queue lag, время до ready,
RSS и другие метрики выбранного трека.
Worker повторно сверяет размер и SHA-256 исходника. Original сохраняется без
перекодирования. Все варианты создаются в отдельном ограниченном процессе:
## Ограничения и следующие шаги
| Вариант | Формат и размеры | Параметры |
| --- | --- | --- |
| thumbnail | JPEG, вписан в 256×256 без увеличения | quality=85, белая подложка alpha |
| crop | JPEG, центральный crop 128×128 | quality=85; маленькое изображение увеличивается |
| optimized | WebP, вписан в 1280×1280 без увеличения | quality=80, method=4, alpha сохраняется |
Сейчас это база для дальнейшей реализации, не завершённая lab5. Очереди и
асинхронной обработки нет; прежние результаты lab4 к этому дереву не относятся.
Сначала просмотр пользователем и выбор трека/брокера, затем реализация;
тестовый прогон и новые evidence — по отдельному указанию после реализации.
Применяется EXIF orientation; EXIF/ICC/XMP и текстовые metadata не переносятся
в варианты. JPEG создаётся из RGB с белой подложкой, WebP — из чистого RGBA.
Алгоритмы используют Pillow 13.0.0. Metadata содержит реальные размеры, bytes и SHA-256.
Лимиты: PNG/JPEG, 10 MiB, 40 млн пикселей; JOB_TIMEOUT_SECONDS=60;
IMAGE_MEMORY_MIB=768 адресного пространства и IMAGE_CPU_SECONDS=20 CPU на процесс.
Контейнер worker: 2 GiB, 2 CPU, tmpfs 128 MiB. Отмена завершает дочерний процесс;
S3 adapter дожидается SDK-потока перед освобождением блокировки/компенсацией.
## Повторы, идемпотентность и очистка
Состояния: queued → processing → ready; временный отказ возвращает queued,
постоянный отказ или исчерпание бюджета переводит failed с безопасным error_code.
JOB_MAX_ATTEMPTS=5 на цикл. Номер attempt монотонен, бюджет закреплён в attempt_limit.
Для попытки n текущего цикла ceiling=min(JOB_RETRY_MAX_SECONDS,
JOB_RETRY_SECONDS × 2^(n−1)); delay равномерно выбирается в [ceiling/2, ceiling].
База — 5 секунд, максимум — 120 секунд. next_attempt_at сохраняется в БД;
дубликат не пересчитывает задержку. Последняя причина остаётся в job даже при
публичном retry_exhausted. Ошибки изображения/исходника/конфигурации постоянные;
timeout, временные ошибки S3/сети и файловой системы повторяются.
Job захватывается с lease_token и JOB_LEASE_SECONDS=90. Publisher восстанавливает
истёкший processing в пределах бюджета. Перед PUT worker блокирует строку job,
проверяет актуальность token и lease, держит блокировку до записи original,
трёх вариантов и COMMIT ready. Reaper пропускает заблокированные строки.
Устаревшая попытка не может начать запись. Фиксированные ключи по
(attachment_id, pipeline_version, variant) исключают второй набор объектов.
Дубль ready подтверждается без обработки и PUT. Это доставка как минимум один
раз с идемпотентным эффектом, не exactly-once выполнение.
WORKER_CONCURRENCY=2, отдельный канал каждого consumer, prefetch=1 на канал.
Publisher и worker используют init:true: Python не является PID 1, поэтому
fault hooks с SIGKILL действительно завершают процесс. SIGTERM отменяет работу
без преждевременного ack. Некорректные envelopes отклоняются без сохранения payload.
Queued/processing staging не удаляется. Для ready retention — 86400 секунд;
для failed — FAILED_STAGING_RETENTION_SECONDS=604800 (семь суток).
Cleanup и replay блокируют одну строку job. Если staging совпадает с готовым
original, объект сохраняется. Частичные конечные объекты failed автоматически
не удаляются; после replay используются те же ключи.
## Реестр failed и операторский replay
Failed registry находится в PostgreSQL. Команды требуют операторского доступа
к Docker/credentials стенда; публичного endpoint для replay нет:
```sh
docker compose run --rm --no-deps worker python -m app.core.job_admin list-failed
docker compose run --rm --no-deps worker python -m app.core.job_admin show JOB_ID
docker compose run --rm --no-deps worker python -m app.core.job_admin replay JOB_ID --request-id REQUEST_UUID --actor operator --reason dependency_restored
docker compose run --rm --no-deps worker python -m app.core.job_admin history JOB_ID
```
JOB_ID/REQUEST_UUID заменить настоящими UUID. Reasons: dependency_restored,
configuration_fixed, manual_review. Actor — аудиторская метка, не система входа.
Replay принимает только failed, сверяет сохранённый исходник по bytes/SHA-256,
атомарно записывает аудит и новый ограниченный бюджет. Счётчик attempt не
сбрасывается, старые envelopes не запускают новый цикл. Повтор того же request-id
с теми же параметрами возвращает already_applied даже после ready; конфликт
параметров отклоняется. После неоднозначного ответа повторяют тот же request-id.
Готовую job с новым request-id повторить нельзя. После удаления staging replay
отклоняется с source_expired; отсутствующий объект даёт source_missing.
## Проверки уровня 3 и 4
Полный воспроизводимый прогон на отдельном стенде:
```sh
make test-grade4 PYTHON=.venv/bin/python
```
Цель последовательно запускает контракт/OpenAPI, тестовый стенд, собственные
тесты с реальными PostgreSQL/Redis, публичный smoke, идемпотентность через две
реплики, persistence/failover, outbox и сценарии worker/replay. Подготовка БД
для pytest выполняется только в выделенной базе *_test, Redis — DB15.
Публичные tests/ и обязательные contracts/ не изменены.
Отдельные сценарии на уже работающем изолированном стенде:
```sh
make test-outbox-recovery BASE_URL=http://localhost:28080
make test-jobs BASE_URL=http://localhost:28080
```
При выборе своего проекта задать COMPOSE_PROJECT_NAME и соответствующие порты
в окружении. Сценарии сверяют BASE_URL с API выбранного Compose-проекта,
создают синтетические данные и возвращают штатные worker/publisher в finally.
Перезапуски сохраняют volumes. Сброса пользовательских данных нет.
Результаты и сопоставление каждому пункту приведены в evidence/lab5-verification.json;
подробные снимки состояний/объектов/retry — в evidence/lab5-jobs.json.
Секреты, tokens и исходные broker payload в evidence не записываются.
## Фактический результат проверок
Все перечисленные проверки завершились успешно на отдельном проекте
`mtusi-lab5-verification`, HTTP http://localhost:28080, без удаления volumes:
| Проверка | Результат |
| --- | --- |
| Собственные тесты PostgreSQL/Redis/API/миграций/изображений/OS-лимитов | 199 passed, без пропусков |
| Дополнительные timeout и failed retention/SHA-256 | 2 passed |
| Контракт, OpenAPI и Compose config | PASS |
| Публичный smoke | PASS |
| 24 конкурентных повтора сообщения через две API-реплики | PASS |
| Публичные persistence и failover | PASS |
| SIGKILL publisher после COMMIT до publish | 137 → queued → ready; без повторного upload |
| Остановленный worker, рестарт API/брокера/хранилищ | ready того же ID; сообщение RabbitMQ сохранилось с volume |
| SIGKILL после четырёх PUT до COMMIT | processing → ready на попытке 2; тот же набор/checksums объектов |
| SIGKILL после COMMIT до ack | ready-дубль подтверждён без изменения metadata/Last-Modified |
| Временная ошибка и replay | failed после 5 попыток, затем ready; повтор request-id безопасен |
| Постоянная ошибка и replay | failed после 1 попытки, затем ready; новый replay ready отклонён |
| Конкуренция/подтверждения RabbitMQ после сценариев | 2 consumer, prefetch=1; ready=0, unacknowledged=0 |
| Неизменность публичных tests/contracts/README и whitespace | PASS |
Суммарно выполнен 201 собственный тест двумя прогонами; новые тесты включены
в последующие полные запуски make test-grade4. Тесты намеренно меняют только
выделенную БД *_test и Redis DB15. API и worker в интеграционных сценариях
используют собственную БД тестового Compose-проекта.
Измеренные задержки пяти неудачных попыток: 4.613, 6.837, 17.323, 30.885,
42.633 секунды; все попали в соответствующие интервалы jitter. Задержка
последней failed-попытки сохраняется для диагностики, автоматический retry
после исчерпания бюджета не выполняется. Ограничение максимума проверено отдельно.
Во время проверки исправлена проблема fault hooks с PID 1 через init:true.
Старые собственные проверки lab4 адаптированы к обязательному переходу
202 queued → ready; публичные проверки не изменены. Цель make test-grade4
переведена на lab5 и последовательные сценарии, чтобы рестарты инфраструктуры
не пересекались с регрессией БД. RabbitMQ используется как брокер, Redis — для сессий.
## Границы результата
Проверки относятся к одному Docker-хосту и рестартам с сохранением volumes.
PostgreSQL, RabbitMQ и MinIO не реплицированы; потеря диска/хоста не покрыта.
Ожидание в очереди не ограничено; admission control и исследование перегрузки
относятся к будущему уровню 5. Pipeline фиксирован на версии 1; изменение
алгоритмов требует решения о новой версии/миграции результатов.
Официальные источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability),
[publisher confirms и consumer acknowledgements](https://www.rabbitmq.com/docs/confirms),
[aio-pika](https://docs.aio-pika.com/quick-start.html),
[Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html),
[Pillow WebP](https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#webp).
+33
View File
@@ -0,0 +1,33 @@
"""Остановка фоновых процессов и безопасные сообщения об отказах."""
import asyncio
import logging
import signal
from aio_pika.exceptions import AMQPException, ChannelInvalidStateError
from sqlalchemy.exc import InterfaceError, OperationalError
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
TRANSIENT_JOB_ERRORS = (AMQPException, ChannelInvalidStateError, InterfaceError, OperationalError, PoolTimeoutError, OSError)
logger = logging.getLogger(__name__)
def shutdown_event() -> asyncio.Event:
stop = asyncio.Event()
loop = asyncio.get_running_loop()
for signum in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(signum, stop.set)
return stop
async def wait_or_stop(stop: asyncio.Event, seconds: float) -> None:
try:
async with asyncio.timeout(seconds):
await stop.wait()
except TimeoutError:
pass
def log_background_error(operation: str, error: Exception) -> None:
# SDK/DB exceptions могут содержать параметры и credentials; выводим только тип.
logger.warning("%s: %s", operation, type(error).__name__)
+7 -3
View File
@@ -6,19 +6,22 @@ from dataclasses import asdict
import json
from app.core.config import Settings
from app.core.jobs import JobSettings
from app.core.database import create_db_engine, create_session_factory
from app.core.s3 import S3Settings
from app.domain.attachments import reconcile_attachments
from app.domain.attachment_jobs import cleanup_staging
from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error
from app.storage.s3 import create_s3_storage
async def cleanup_once(factory, storage, apply: bool) -> bool:
async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool:
completed = True
try:
async with factory() as session:
result = await reconcile_attachments(session, storage, dry_run=not apply)
print(json.dumps(asdict(result)), flush=True)
staging = await cleanup_staging(session, storage, jobs, apply=apply)
print(json.dumps({**asdict(result), "terminal_staging": staging}), flush=True)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error("Reconciliation", error)
completed = False
@@ -28,12 +31,13 @@ async def cleanup_once(factory, storage, apply: bool) -> bool:
async def run(once: bool, apply: bool) -> None:
settings = Settings() # type: ignore
s3_settings = S3Settings()
jobs = JobSettings()
engine = create_db_engine(settings)
factory = create_session_factory(engine)
storage = create_s3_storage(s3_settings)
try:
while True:
completed = await cleanup_once(factory, storage, apply)
completed = await cleanup_once(factory, storage, apply, jobs)
if once:
if not completed:
raise SystemExit(1)
+14
View File
@@ -0,0 +1,14 @@
"""Качество вариантов и пределы отдельного процесса обработки."""
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
class ImageSettings(BaseSettings):
model_config = SettingsConfigDict(env_prefix="IMAGE_", env_file=".env", extra="ignore")
jpeg_quality: int = Field(default=85, ge=1, le=95)
webp_quality: int = Field(default=80, ge=1, le=100)
webp_method: int = Field(default=4, ge=0, le=6)
memory_mib: int = Field(default=768, ge=256, le=1024)
cpu_seconds: int = Field(default=20, ge=1, le=120)
+65
View File
@@ -0,0 +1,65 @@
"""Операторский интерфейс failed registry. Доступ требует credentials стенда."""
import argparse
import asyncio
import json
from uuid import UUID
from app.core.background import log_background_error
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
from app.domain.job_admin import REPLAY_REASONS, history, list_failed, replay_failed_job, show_job
from app.errors.exceptions import JobOperationError
from app.storage.s3 import create_s3_storage
async def execute(args):
settings = Settings()
engine = create_db_engine(settings)
storage = None
try:
factory = create_session_factory(engine)
async with factory() as session:
if args.command == "list-failed":
result = await list_failed(session, args.limit)
elif args.command == "show":
result = await show_job(session, args.job_id)
elif args.command == "history":
result = await history(session, args.job_id)
else:
storage = create_s3_storage(settings.s3)
result = await replay_failed_job(session, storage, JobSettings(), args.job_id,
request_id=args.request_id, actor=args.actor, reason=args.reason)
print(json.dumps(result, default=str), flush=True)
finally:
if storage is not None:
await storage.aclose()
await engine.dispose()
def main():
parser = argparse.ArgumentParser(description="Реестр failed и явный replay с аудитом")
commands = parser.add_subparsers(dest="command", required=True)
listing = commands.add_parser("list-failed")
listing.add_argument("--limit", type=int, choices=range(1, 501), default=50, metavar="1..500")
for name in ("show", "history", "replay"):
command = commands.add_parser(name)
command.add_argument("job_id", type=UUID)
if name == "replay":
command.add_argument("--request-id", type=UUID, required=True)
command.add_argument("--actor", required=True)
command.add_argument("--reason", choices=REPLAY_REASONS, required=True)
args = parser.parse_args()
try:
asyncio.run(execute(args))
except JobOperationError as error:
print(json.dumps({"error_code": error.code}), flush=True)
raise SystemExit(2) from None
except Exception as error:
log_background_error("Job administration failed", error)
raise SystemExit(1) from None
if __name__ == "__main__":
main()
+133 -13
View File
@@ -1,17 +1,137 @@
"""Заготовка отдельного worker lab5, не одноразового валидатора image_worker.
"""Отдельные consumers с ограниченной конкуренцией; API не выполняет resize."""
Здесь будут startup/shutdown ресурсов, цикл получения заданий, ограниченная
concurrency/prefetch, восстановление pending и остановка с сохранением работы.
Каждая задача получает отдельную AsyncSession. Обработка изображений идёт
в процессе с лимитами памяти/CPU/времени; CancelledError не подавляется.
В Compose и API модуль пока не подключён.
"""
import argparse
import asyncio
from dataclasses import dataclass
import json
import logging
import os
import signal
from uuid import UUID, uuid4
from PIL import features
from pydantic import ValidationError
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
from app.domain.attachment_jobs import process_attachment_job
from app.errors.exceptions import ProcessingError
from app.schemas.job import JobMessage
from app.storage.broker import JobBroker
from app.storage.s3 import create_s3_storage
async def run_worker() -> None:
"""Запустить consumer с lease/retry, ограничением ресурсов и корректным ack.
@dataclass(frozen=True)
class Faults:
"""Только явные CLI-флаги; обычный worker не включает инъекции сбоев."""
SIGTERM должен завершить или оставить восстанавливаемую попытку; остановка
worker не заставляет API выполнять resize. Инициализация ресурсов — здесь.
"""
raise NotImplementedError("Lab5: отдельный worker ещё не реализован")
crash_after_object: UUID | None = None
crash_before_ack: UUID | None = None
fail_transient: UUID | None = None
fail_permanent: UUID | None = None
object_name: str = "optimized"
def kill(self, attachment_id: UUID, phase: str) -> None:
print(json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), flush=True)
os.kill(os.getpid(), signal.SIGKILL)
def after_object(self, attachment_id: UUID, name: str) -> None:
if attachment_id == self.crash_after_object and name == self.object_name:
self.kill(attachment_id, "after_object:" + name)
def before_ack(self, attachment_id: UUID) -> None:
if attachment_id == self.crash_before_ack:
self.kill(attachment_id, "before_ack")
def before_processing(self, attachment_id: UUID) -> None:
if attachment_id == self.fail_transient:
raise TimeoutError("injected transient failure")
if attachment_id == self.fail_permanent:
raise ProcessingError("injected_permanent")
async def consume_connected(factory, storage, broker: JobBroker, settings: JobSettings, stop: asyncio.Event, faults: Faults) -> None:
consumer = uuid4().hex
while not stop.is_set():
try:
await broker.ensure_ready()
delivery = await broker.receive(consumer)
if delivery is None:
continue
try:
message = JobMessage.model_validate_json(delivery.payload)
except ValidationError:
await broker.reject_invalid(delivery.id)
continue
try:
async with factory() as session:
acknowledge = await process_attachment_job(
session, storage, message, settings,
after_object=faults.after_object, before_processing=faults.before_processing,
)
except ProcessingError:
await broker.reject_invalid(delivery.id)
else:
if acknowledge:
faults.before_ack(message.attachment_id)
await broker.acknowledge_job(delivery.id)
else:
await broker.defer_job(delivery.id)
except TRANSIENT_JOB_ERRORS as error:
log_background_error("Worker delivery deferred", error)
await broker.aclose()
await wait_or_stop(stop, settings.publisher_poll_seconds)
async def consume(factory, storage, settings, stop, faults) -> None:
broker = JobBroker(settings)
try:
await consume_connected(factory, storage, broker, settings, stop, faults)
finally:
await broker.aclose()
async def run_worker(faults: Faults) -> None:
settings = Settings()
jobs = JobSettings()
logging.basicConfig(level=settings.log_level)
if not features.check("webp"):
logging.error("Pillow requires WebP support for the image worker")
raise RuntimeError("WebP codec is unavailable")
stop = shutdown_event()
engine = create_db_engine(settings)
storage = None
try:
storage = create_s3_storage(settings.s3)
factory = create_session_factory(engine)
async with asyncio.TaskGroup() as group:
tasks = [group.create_task(consume(factory, storage, jobs, stop, faults)) for _ in range(jobs.worker_concurrency)]
await stop.wait()
# Отмена убивает resize-процесс, дожидается SDK и не отправляет ack.
for task in tasks:
task.cancel()
finally:
if storage is not None:
await storage.aclose()
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description="Image worker с явными fault-injection hooks")
modes = parser.add_mutually_exclusive_group()
for flag in ("crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent"):
modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID")
parser.add_argument("--object-name", choices=("original", "thumbnail", "crop", "optimized"), default="optimized")
args = parser.parse_args()
faults = Faults(**vars(args))
try:
asyncio.run(run_worker(faults))
except Exception as error:
log_background_error("Worker stopped", error)
raise SystemExit(1) from None
if __name__ == "__main__":
main()
+35
View File
@@ -0,0 +1,35 @@
"""Настройки очереди из .env; credentials брокера читаются отдельно из файла."""
from pathlib import Path
from typing import Self
from pydantic import Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
class JobSettings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
broker_host: str = "broker"
broker_port: int = Field(default=5672, ge=1, le=65535)
broker_password_file: Path = Path("/run/secrets/broker_password")
broker_queue: str = "attachment-jobs"
broker_user: str = "messenger"
broker_vhost: str = "/"
worker_concurrency: int = Field(default=2, ge=1, le=4)
job_max_attempts: int = Field(default=5, ge=1, le=100)
job_timeout_seconds: int = Field(default=60, ge=10, le=300)
job_lease_seconds: int = Field(default=90, ge=20, le=600)
job_retry_seconds: int = Field(default=5, ge=1, le=300)
job_retry_max_seconds: int = Field(default=120, ge=1, le=3600)
failed_staging_retention_seconds: int = Field(default=604800, ge=300)
publisher_poll_seconds: float = Field(default=1, gt=0, le=10)
staging_retention_seconds: int = Field(default=86400, ge=300)
@model_validator(mode="after")
def validate_lease(self) -> Self:
if self.job_lease_seconds < self.job_timeout_seconds + 30:
raise ValueError("Lease должен превышать timeout задания минимум на 30 секунд")
if self.job_retry_max_seconds < self.job_retry_seconds:
raise ValueError("Retry maximum must be at least the base delay")
return self
+66 -11
View File
@@ -1,15 +1,70 @@
"""Заготовка процесса доставки PostgreSQL outbox в durable broker.
"""Доставка намерений из PostgreSQL; crash hook включается только явно через CLI."""
Не держать транзакцию/блокировки во время бесконечного сетевого ожидания.
Согласовать lease publisher, ограниченные пачки и повтор после COMMIT/publish.
Процесс пока не подключён к Compose или жизненному циклу API.
"""
import argparse
import asyncio
import json
import logging
import os
import signal
from uuid import UUID
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
from app.domain.attachment_jobs import publish_next_job, recover_expired_jobs
from app.storage.broker import JobBroker
async def run_publisher() -> None:
"""Публиковать сохранённые намерения и фиксировать подтверждение брокера.
def crash_before_publish(attachment_id: UUID) -> None:
"""Реальный SIGKILL после чтения принятой job, до publisher confirm и отметки publish."""
print(json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), flush=True)
os.kill(os.getpid(), signal.SIGKILL)
После crash между publish и отметкой допустим дубль того же job_id.
Недоступность брокера не должна терять уже принятые задания.
"""
raise NotImplementedError("Lab5: outbox publisher ещё не реализован")
async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
settings = Settings()
jobs = JobSettings()
logging.basicConfig(level=settings.log_level)
stop = shutdown_event()
engine = create_db_engine(settings)
broker = None
try:
broker = JobBroker(jobs)
factory = create_session_factory(engine)
while not stop.is_set():
published = False
try:
await broker.ensure_ready()
async with factory() as session:
await recover_expired_jobs(session, jobs)
published = await publish_next_job(
session, broker, attachment_id=crash_attachment_id,
before_publish=crash_before_publish if crash_attachment_id is not None else None,
)
except TRANSIENT_JOB_ERRORS as error:
log_background_error("Publisher deferred", error)
if not published:
await wait_or_stop(stop, jobs.publisher_poll_seconds)
finally:
if broker is not None:
await broker.aclose()
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description="Публикация устойчивых jobs в RabbitMQ")
parser.add_argument(
"--crash-before-publish", type=UUID, metavar="ATTACHMENT_ID",
help="аварийно завершить этот процесс перед публикацией только указанного вложения",
)
args = parser.parse_args()
try:
asyncio.run(run_publisher(args.crash_before_publish))
except Exception as error:
log_background_error("Publisher stopped", error)
raise SystemExit(1) from None
if __name__ == "__main__":
main()
+28
View File
@@ -0,0 +1,28 @@
"""Один pipeline изображений в Linux-процессе с лимитами памяти и CPU."""
from pathlib import Path
import resource
import sys
from app.core.image_processing import ImageSettings
from app.domain.image_pipeline import create_variants
from app.errors.exceptions import IMAGE_WORKER_ERRORS, pipeline_worker_error_code
def main() -> int:
settings = ImageSettings()
memory = settings.memory_mib * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
resource.setrlimit(resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds))
source, directory, metadata, content_type, maximum = sys.argv[1:]
code = 0
try:
result = create_variants(Path(source), Path(directory), content_type, int(maximum), settings)
Path(metadata).write_text(result.model_dump_json())
except IMAGE_WORKER_ERRORS as error:
code = pipeline_worker_error_code(error)
return code
if __name__ == "__main__":
raise SystemExit(main())
+186 -33
View File
@@ -1,45 +1,198 @@
"""Заготовки сценариев lab5; не подключены к upload или worker.
"""Приём, durable публикация, попытки и фиксация результата обработки."""
Здесь появятся транзакционные границы и бизнес-условия; SQL и SDK остаются
в storage/. Одна AsyncSession принадлежит одной последовательной задаче.
"""
from uuid import UUID
import asyncio
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
from uuid import UUID, uuid4
from minio.error import MinioException
from sqlalchemy.ext.asyncio import AsyncSession
from urllib3.exceptions import HTTPError as StorageHTTPError
from app.core.jobs import JobSettings
from app.domain.job_retry import classify_failure, retry_delay
from app.domain.job_files import fetch_original, render_variants
from app.errors.exceptions import ProcessingError
from app.schemas.attachment import VARIANT_FILENAMES
from app.schemas.job import JobMessage
from app.storage import jobs, outbox
from app.storage.broker import JobBroker
from app.storage.jobs import JobWork
from app.storage.s3 import MinioStorage
async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID) -> UUID:
"""Создать job и outbox в транзакции queued metadata после durable staging.
Возвращать стабильный job_id. Commit принадлежит сценарию приёма upload;
202 допустим только после устойчивого оригинала и успешного commit.
"""
raise NotImplementedError("Lab5: постановка задания ещё не реализована")
@dataclass(frozen=True)
class Claim:
work: JobWork | None = None
acknowledge: bool = False
async def process_attachment_job(session: AsyncSession, job_id: UUID) -> None:
"""Захватить job с lease и опубликовать original и обязательные варианты.
Ready-дубль не меняет результат. Lease и версия защищают от устаревшей
попытки. S3/metadata фиксируются до ack; отмена не подавляется.
"""
raise NotImplementedError("Lab5: обработка задания ещё не реализована")
async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
"""Включить намерение обработки в транзакцию queued metadata; без commit."""
return await jobs.create_job(session, attachment_id, staging_key)
async def recover_expired_jobs(session: AsyncSession) -> int:
"""Вернуть зависший processing в очередь после истечения lease.
Учитывать лимит попыток и защиту от конкурентного завершения старым worker.
Точное правило heartbeat/deadline определяется до реализации.
"""
raise NotImplementedError("Lab5: восстановление lease ещё не реализовано")
async def publish_next_job(
session: AsyncSession, broker: JobBroker, *,
attachment_id: UUID | None = None,
before_publish: Callable[[UUID], None] | None = None,
) -> bool:
"""Отметка publish следует за durable publisher confirm; crash допускает тот же job-дубль."""
job = await outbox.next_unpublished_job(session, attachment_id=attachment_id)
published = job is not None
if job is not None:
if before_publish is not None:
before_publish(job.attachment_id)
await broker.publish_job(JobMessage(
job_id=job.id, attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version, attempt=job.attempt + 1,
created_at=job.created_at,
))
outbox.mark_published(job)
await session.commit()
return published
async def replay_failed_job(session: AsyncSession, job_id: UUID) -> None:
"""Явно повторить только failed job, сохранив аудит и ключ логического эффекта.
async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSettings) -> Claim:
"""Дубликат не захватывает живую аренду и не откатывает terminal-состояние."""
row = await jobs.lock_job(session, message.job_id)
result = Claim()
if row is None:
raise ProcessingError("unknown_job")
job, attachment = row
if message.attachment_id != attachment.id or message.pipeline_version != job.pipeline_version:
raise ProcessingError("invalid_job")
now = await jobs.database_time(session)
if job.attempt_limit is None:
job.attempt_limit = settings.job_max_attempts
if attachment.status in {"ready", "failed"}:
result = Claim(acknowledge=True)
elif message.attempt <= job.attempt and attachment.status == "queued":
# Предыдущая попытка уже устойчиво заменена новой в PostgreSQL.
result = Claim(acknowledge=True)
elif message.attempt < job.attempt:
result = Claim(acknowledge=True)
elif message.attempt > job.attempt + 1:
raise ProcessingError("invalid_attempt")
elif attachment.status == "queued" and job.next_attempt_at <= now:
if job.attempt >= (job.attempt_limit or settings.job_max_attempts) or job.staging_key is None:
jobs.record_failure(job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0)
result = Claim(acknowledge=True)
else:
result = Claim(work=jobs.start_job(job, attachment, uuid4(), now, settings.job_lease_seconds))
await session.commit()
return result
Не переисполнять ready. Проверка допустимости, лимиты повторов и способ
операторского вызова будут определены вместе с реестром failed/DLQ.
"""
raise NotImplementedError("Lab5: replay ещё не реализован")
async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettings, code: str, *, permanent: bool) -> None:
# Неоднозначный COMMIT проверяется повторным чтением: ready сохраняется.
await session.rollback()
row = await jobs.lock_job(session, work.job_id)
if row is not None:
job, attachment = row
if attachment.status == "processing" and job.lease_token == work.lease_token:
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
jobs.record_failure(
job, attachment, await jobs.database_time(session),
"retry_exhausted" if exhausted and not permanent else code,
terminal=permanent or exhausted,
retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
)
job.last_error_code = code
job.last_error_permanent = permanent
await session.commit()
async def build_result(
session: AsyncSession, storage: MinioStorage, work: JobWork,
after_object: Callable[[UUID, str], None] | None = None,
) -> bool:
"""Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки."""
with TemporaryDirectory(prefix="messenger-job-") as directory:
source = Path(directory) / "original"
await fetch_original(storage, work, source)
result = await render_variants(storage, work, source, Path(directory))
row = await jobs.lock_job(session, work.job_id)
if row is None:
await session.rollback()
return True
job, attachment = row
now = await jobs.database_time(session)
if attachment.status != "processing" or job.lease_token != work.lease_token:
await session.rollback()
return True
if job.lease_until is None or job.lease_until <= now:
await session.rollback()
return False
# Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED.
# При отмене S3 adapter дожидается SDK-потока до освобождения блокировки.
await storage.put_file(
work.original_key, source, size_bytes=work.size_bytes,
content_type=work.content_type, sha256=work.sha256,
)
if after_object is not None:
after_object(work.attachment_id, "original")
for variant in result.variants:
await storage.put_file(
work.variant_key(variant.name), Path(directory) / VARIANT_FILENAMES[variant.name],
size_bytes=variant.size_bytes, content_type=variant.content_type, sha256=variant.sha256,
)
if after_object is not None:
after_object(work.attachment_id, variant.name)
jobs.complete_job(job, attachment, work, [variant.model_dump() for variant in result.variants], await jobs.database_time(session))
await session.commit()
return True
async def process_attachment_job(
session: AsyncSession, storage: MinioStorage, message: JobMessage, settings: JobSettings,
*, after_object: Callable[[UUID, str], None] | None = None,
before_processing: Callable[[UUID], None] | None = None,
) -> bool:
claim = await claim_job(session, message, settings)
acknowledge = claim.acknowledge
if claim.work is not None:
work = claim.work
try:
async with asyncio.timeout(settings.job_timeout_seconds):
if before_processing is not None:
before_processing(work.attachment_id)
acknowledge = await build_result(session, storage, work, after_object)
except (ProcessingError, MinioException, StorageHTTPError, OSError, TimeoutError) as error:
code, permanent = classify_failure(error)
await save_failure(session, work, settings, code, permanent=permanent)
acknowledge = True
# Ошибка БД/неизвестная ошибка или отмена не подтверждают доставку.
return acknowledge
async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) -> int:
rows = await jobs.expired_jobs(session)
now = await jobs.database_time(session)
for job, attachment in rows:
if job.attempt_limit is None:
job.attempt_limit = settings.job_max_attempts
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
jobs.record_failure(
job, attachment, now, "retry_exhausted" if exhausted else "worker_timeout",
terminal=exhausted, retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
)
job.last_error_code = "worker_timeout"
job.last_error_permanent = False
await session.commit()
return len(rows)
async def cleanup_staging(session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool) -> int:
rows = await jobs.terminal_staging(session, settings.staging_retention_seconds, settings.failed_staging_retention_seconds)
if apply:
for job, attachment in rows:
if job.staging_key != attachment.object_key:
await storage.delete_object(job.staging_key)
jobs.clear_staging(job)
await session.commit()
else:
await session.rollback()
return len(rows)
+36 -19
View File
@@ -13,6 +13,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.core.request_capacity import RequestCapacity
from app.domain.chats import ensure_chat_member
from app.domain.attachment_jobs import enqueue_attachment_job
from app.domain.images import validate_filename, validate_image
from app.errors.exceptions import (
conflict,
@@ -28,7 +29,7 @@ from app.errors.handlers import (
log_recoverable_error,
operation_timeout,
)
from app.schemas.attachment import Attachment
from app.schemas.attachment import Attachment, VARIANT_FILENAMES
from app.schemas.user import User
from app.storage import attachments as repository
from app.storage.s3 import MinioStorage
@@ -151,7 +152,7 @@ async def recover_upload(
await session.rollback()
async def publish_original(
async def stage_attachment(
session: AsyncSession,
storage: MinioStorage,
user: User,
@@ -161,8 +162,9 @@ async def publish_original(
filename: str,
content_type: str,
) -> Attachment:
"""Сохранить исходник в staging и атомарно принять metadata/job."""
id = uuid4()
key = f"originals/{uuid4().hex}"
key = f"staging/{uuid4().hex}"
# Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT.
await repository.reserve_upload(session, id, key)
await session.commit()
@@ -181,13 +183,16 @@ async def publish_original(
id=id,
chat_id=chat_id,
owner_id=user.id,
object_key=key,
object_key=f"originals/{id.hex}/v1",
status="queued",
filename=filename,
content_type=content_type,
size_bytes=original.size_bytes,
sha256=original.sha256,
)
await enqueue_attachment_job(session, id, key)
await repository.finish_upload(session, id)
# Снимок queued строится до COMMIT, даже если worker очень быстрый.
result = Attachment.model_validate(record)
await session.commit()
except BaseException:
@@ -223,7 +228,7 @@ async def upload_attachment(
settings.chunk_bytes,
)
await validate_image(original.path, content_type, settings)
result = await publish_original(
result = await stage_attachment(
session,
storage,
user,
@@ -245,6 +250,13 @@ async def get_attachment(
return Attachment.model_validate(record)
@dataclass(frozen=True)
class DownloadInfo:
filename: str
content_type: str
size_bytes: int
async def download_attachment(
session: AsyncSession,
storage: MinioStorage | None,
@@ -252,25 +264,30 @@ async def download_attachment(
attachment_id: UUID,
*,
variant: str = "original",
) -> tuple[Attachment, AsyncIterator[bytes]]:
) -> tuple[DownloadInfo, AsyncIterator[bytes]]:
result = await get_attachment(session, user, attachment_id)
if variant != "original":
raise not_found()
if result.status != "ready":
raise conflict("Вложение пока недоступно для скачивания")
storage = require_storage(storage)
record = await repository.get_attachment(session, attachment_id)
if record is None:
raise not_found()
key = record.object_key
if variant == "original":
key = record.object_key
info = DownloadInfo(result.filename, result.content_type, result.size_bytes)
else:
selected = next((item for item in result.variants if item.name == variant), None)
keys = {"thumbnail": record.thumbnail_key, "crop": record.crop_key, "optimized": record.optimized_key}
key = keys.get(variant)
if selected is None or key is None:
raise not_found()
info = DownloadInfo(VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes)
await session.rollback()
info = await storage.stat_object(key)
if (
info is None
or info.size_bytes != result.size_bytes
or info.content_type != result.content_type
):
raise unavailable("Оригинал временно недоступен")
stored = await storage.stat_object(key)
if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type:
raise unavailable("Файл временно недоступен")
stream = await storage.open_object(key)
return result, stream
return info, stream
async def validate_message_attachments(
@@ -283,8 +300,8 @@ async def validate_message_attachments(
records = await repository.get_for_chat(session, chat_id, attachment_ids)
if len(records) != len(attachment_ids):
raise not_found()
if any(record.status != "ready" for record in records):
raise conflict("Вложение ещё не готово")
if any(record.status == "failed" for record in records):
raise conflict("Обработка вложения завершилась ошибкой")
async def reconcile_attachments(
+70 -25
View File
@@ -1,34 +1,79 @@
"""Заготовки преобразований lab5, без доступа к БД, брокеру и HTTP.
Pipeline запускается внутри ограниченного процесса worker. Исходные bytes
не перекодируются; варианты удаляют метаданные по документированной политике.
Значения качества JPEG/WebP предстоит выбрать и зафиксировать в REPORT.md.
"""
"""Thumbnail, crop и WebP; один ограниченный процесс, последовательная обработка."""
import hashlib
from pathlib import Path
from PIL import Image, ImageOps
def create_thumbnail(source: Path, destination: Path) -> None:
"""Создать JPEG, вписанный в 256×256 без увеличения.
Применить EXIF orientation, alpha положить на белый; округлять размеры
вниз, минимум 1 px. Публичный fixture 320×200 должен дать 256×160.
"""
raise NotImplementedError("Lab5: thumbnail ещё не реализован")
from app.core.image_processing import ImageSettings
from app.domain.image_validation import decode_image
from app.schemas.attachment import PipelineResult, Variant
def create_crop(source: Path, destination: Path) -> None:
"""Создать центрированный квадрат JPEG 128×128; увеличение допустимо.
Учитывать EXIF orientation и прозрачность по той же политике, что thumbnail.
"""
raise NotImplementedError("Lab5: crop ещё не реализован")
def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]:
"""Вписать без увеличения, округляя вниз, минимум один пиксель."""
longest = max(size)
result = size
if longest > maximum:
result = (max(1, size[0] * maximum // longest), max(1, size[1] * maximum // longest))
return result
def create_optimized(source: Path, destination: Path) -> None:
"""Создать WebP, вписанный в 1280×1280 без увеличения.
def describe_variant(name: str, content_type: str, path: Path, size: tuple[int, int]) -> Variant:
checksum = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(65536), b""):
checksum.update(chunk)
return Variant(
name=name, content_type=content_type, size_bytes=path.stat().st_size,
width=size[0], height=size[1], sha256=checksum.hexdigest(),
)
Удалить метаданные и применить описанную политику качества. Уменьшение
размера любого маленького исходника не гарантируется контрактом.
"""
raise NotImplementedError("Lab5: optimized ещё не реализован")
def save_jpeg(image: Image.Image, destination: Path, quality: int) -> None:
"""Белая подложка и новый image без EXIF/ICC/XMP/текстовых метаданных."""
with Image.new("RGB", image.size, "white") as clean:
with image.getchannel("A") as alpha:
clean.paste(image, mask=alpha)
clean.save(destination, format="JPEG", quality=quality, exif=b"")
def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Variant:
size = fitted_size(image.size, 256)
with image.resize(size, Image.Resampling.LANCZOS) as resized:
save_jpeg(resized, destination, quality)
return describe_variant("thumbnail", "image/jpeg", destination, size)
def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant:
"""Центрированный JPEG 128×128; маленький исходник допускает увеличение."""
with ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) as crop:
save_jpeg(crop, destination, quality)
return describe_variant("crop", "image/jpeg", destination, (128, 128))
def create_optimized(image: Image.Image, destination: Path, settings: ImageSettings) -> Variant:
"""WebP ≤1280×1280 без увеличения; alpha сохраняется, метаданные удаляются."""
size = fitted_size(image.size, 1280)
with image.resize(size, Image.Resampling.LANCZOS) as resized:
with Image.new("RGBA", size) as clean:
clean.paste(resized)
clean.save(
destination, format="WEBP", lossless=False,
quality=settings.webp_quality, method=settings.webp_method,
exif=b"", icc_profile=b"", xmp=b"",
)
return describe_variant("optimized", "image/webp", destination, size)
def create_variants(source: Path, directory: Path, content_type: str, max_pixels: int, settings: ImageSettings) -> PipelineResult:
decode_image(source, content_type, max_pixels)
with Image.open(source, formats=["PNG", "JPEG"]) as image:
ImageOps.exif_transpose(image, in_place=True)
with image.convert("RGBA") as pixels:
variants = [
create_thumbnail(pixels, directory / "thumbnail.jpg", settings.jpeg_quality),
create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality),
create_optimized(pixels, directory / "optimized.webp", settings),
]
return PipelineResult(variants=variants)
+93
View File
@@ -0,0 +1,93 @@
"""Replay только failed с проверенным исходником и ограниченным новым бюджетом."""
import asyncio
import re
from pathlib import Path
from tempfile import TemporaryDirectory
from uuid import uuid4
from minio.error import S3Error
from app.domain.job_files import fetch_original
from app.errors.exceptions import JobOperationError, ProcessingError
from app.storage import job_admin, jobs
from app.storage.jobs import JobWork
REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review")
def snapshot(job, attachment):
return dict(job_id=str(job.id), attachment_id=str(attachment.id), status=attachment.status,
attempt=job.attempt, attempt_limit=job.attempt_limit, cycle_start_attempt=job.cycle_start_attempt,
error_code=attachment.error_code, last_error_code=job.last_error_code,
last_error_permanent=job.last_error_permanent, staging_retained=job.staging_key is not None,
next_attempt_at=job.next_attempt_at, lease_until=job.lease_until, finished_at=job.finished_at)
async def list_failed(session, limit):
return [snapshot(job, attachment) for job, attachment in await job_admin.failed_jobs(session, limit)]
async def show_job(session, job_id):
row = await jobs.lock_job(session, job_id)
if row is None:
raise JobOperationError("unknown_job")
result = snapshot(*row)
await session.rollback()
return result
async def history(session, job_id):
return [dict(request_id=str(row.id), actor=row.actor, reason=row.reason,
previous_attempt=row.previous_attempt, attempt_limit=row.attempt_limit,
previous_error_code=row.previous_error_code, created_at=row.created_at)
for row in await job_admin.replay_history(session, job_id)]
def same_request(record, job_id, actor, reason):
if record.job_id != job_id or record.actor != actor or record.reason != reason:
raise JobOperationError("request_conflict")
return {"request_id": str(record.id), "job_id": str(job_id), "already_applied": True}
async def replay_failed_job(session, storage, settings, job_id, *, request_id, actor, reason):
if re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None or reason not in REPLAY_REASONS:
raise JobOperationError("invalid_audit_fields")
row = await jobs.lock_job(session, job_id)
if row is None:
raise JobOperationError("unknown_job")
record = await job_admin.replay_record(session, request_id)
if record is not None:
result = same_request(record, job_id, actor, reason)
await session.rollback()
return result
job, attachment = row
if attachment.status != "failed":
raise JobOperationError("job_not_failed")
if job.staging_key is None:
raise JobOperationError("source_expired")
# Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT.
work = JobWork(job.id, attachment.id, uuid4(), job.pipeline_version, job.staging_key,
attachment.content_type, attachment.size_bytes, attachment.sha256)
try:
async with asyncio.timeout(settings.job_timeout_seconds):
with TemporaryDirectory(prefix="replay-source-") as directory:
await fetch_original(storage, work, Path(directory) / "original")
except ProcessingError as error:
raise JobOperationError(error.code) from None
except S3Error as error:
if error.code in {"NoSuchKey", "NoSuchObject"}:
raise JobOperationError("source_missing") from None
raise
limit = job.attempt + settings.job_max_attempts
inserted = await job_admin.insert_replay(
session, id=request_id, job_id=job_id, actor=actor, reason=reason,
previous_attempt=job.attempt, attempt_limit=limit, previous_error_code=job.last_error_code,
)
if not inserted:
result = same_request(await job_admin.replay_record(session, request_id), job_id, actor, reason)
await session.rollback()
return result
job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit)
await session.commit()
return {"request_id": str(request_id), "job_id": str(job_id), "already_applied": False, "status": "queued", "attempt_limit": limit}
+61
View File
@@ -0,0 +1,61 @@
"""Получение исходника и запуск изолированного процесса обработки вариантов."""
import asyncio
import errno
import hashlib
from pathlib import Path
import sys
from anyio import CancelScope
from app.errors.exceptions import ProcessingError
from app.schemas.attachment import PipelineResult
from app.storage.jobs import JobWork
from app.storage.s3 import MinioStorage
async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path) -> None:
if work.size_bytes > storage.settings.max_upload_bytes:
raise ProcessingError("image_limits")
checksum = hashlib.sha256()
size = 0
stream = await storage.open_object(work.staging_key)
try:
with destination.open("wb") as target:
async for chunk in stream:
size += len(chunk)
if size > work.size_bytes:
raise ProcessingError("source_mismatch")
target.write(chunk)
checksum.update(chunk)
finally:
await stream.aclose()
if size != work.size_bytes or checksum.hexdigest() != work.sha256:
raise ProcessingError("source_mismatch")
async def render_variants(storage: MinioStorage, work: JobWork, source: Path, directory: Path) -> PipelineResult:
metadata = directory / "variants.json"
process = await asyncio.create_subprocess_exec(
sys.executable, "-m", "app.core.thumbnail_worker", str(source), str(directory),
str(metadata), work.content_type, str(storage.settings.max_image_pixels),
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
)
try:
code = await process.wait()
finally:
if process.returncode is None:
process.kill()
with CancelScope(shield=True):
await process.wait()
if code == 2:
raise ProcessingError("invalid_image")
if code == 3 or code < 0:
raise ProcessingError("image_limits")
if code == 4:
raise OSError(errno.EIO, "Pipeline filesystem unavailable")
if code == 5:
raise ProcessingError("processing_configuration")
if code != 0:
raise RuntimeError("Image pipeline subprocess failed")
return PipelineResult.model_validate_json(metadata.read_text())
+32
View File
@@ -0,0 +1,32 @@
"""Ограниченный exponential backoff: jitter сохраняется вместе с новой попыткой."""
import errno
import random
from minio.error import S3Error
from app.core.jobs import JobSettings
from app.errors.exceptions import ProcessingError
def retry_delay(settings: JobSettings, cycle_attempt: int) -> float:
ceiling = min(settings.job_retry_max_seconds, settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)))
return random.uniform(ceiling / 2, ceiling)
def classify_failure(error: Exception) -> tuple[str, bool]:
if isinstance(error, ProcessingError):
return error.code, True
if isinstance(error, S3Error):
if error.code in {"NoSuchKey", "NoSuchObject"}:
return "source_missing", True
if error.code in {"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "NoSuchBucket"}:
return "storage_configuration", True
if error.code in {"InternalError", "SlowDown", "ServiceUnavailable", "RequestTimeout", "Throttling"}:
return "storage_unavailable", False
return "storage_rejected", True
if isinstance(error, TimeoutError):
return "processing_timeout", False
if isinstance(error, OSError) and error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}:
return "processing_configuration", True
return "processing_unavailable", False
+1 -1
View File
@@ -15,7 +15,7 @@ from app.schemas.attachment import Attachment
router = APIRouter(prefix='/api/v1', tags=['attachments'])
@router.post('/chats/{chat_id}/attachments', status_code=201)
@router.post('/chats/{chat_id}/attachments', status_code=202)
async def upload(
request: Request, chat_id: UUID, user: CurrentUser, session: DbSession,
storage: S3Storage, capacity: UploadCapacity,
+27
View File
@@ -1,3 +1,5 @@
import errno
from collections.abc import Mapping
from PIL import Image, UnidentifiedImageError
@@ -107,3 +109,28 @@ def image_validation_error(code: int) -> ApiError:
if code == 3 or code < 0:
error = payload_too_large("Превышен лимит пикселей или ресурсов изображения")
return error
class ProcessingError(Exception):
"""Постоянная ошибка pipeline с безопасным машинным кодом."""
def __init__(self, code: str) -> None:
super().__init__(code)
self.code = code
class JobOperationError(Exception):
"""Безопасный код отказа операторской команды."""
def __init__(self, code: str):
self.code = code
super().__init__(code)
def pipeline_worker_error_code(error: Exception) -> int:
"""Отделить ошибки файловой системы от ошибки декодирования изображения."""
if isinstance(error, OSError) and error.errno is not None:
if error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}:
return 5
return 4
return image_worker_error_code(error)
+1 -1
View File
@@ -45,7 +45,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
await redis.aclose()
await engine.dispose()
application = FastAPI(title="Messenger", version="1.4.0", lifespan=lifespan)
application = FastAPI(title="Messenger", version="1.5.0", lifespan=lifespan)
application.state.settings = settings
application.state.engine = engine
+3 -1
View File
@@ -1,4 +1,6 @@
from app.models.job_replay import JobReplay
from app.models.base import Base
from app.models.job import AttachmentJob
from app.models.attachment import Attachment, MessageAttachment, UploadIntent
from app.models.chat import Chat
from app.models.chat_member import ChatMember
@@ -7,4 +9,4 @@ from app.models.idempotency_key import IdempotencyKey
from app.models.message import Message
from app.models.user import User
__all__ = ["Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"]
__all__ = ["JobReplay", "AttachmentJob", "Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"]
+11 -4
View File
@@ -3,7 +3,8 @@
import uuid
from datetime import datetime
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func, text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
@@ -17,7 +18,8 @@ class Attachment(Base):
"content_type IN ('image/png', 'image/jpeg')", name="content_type"
),
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
CheckConstraint("status = 'ready'", name="status"),
CheckConstraint("status IN ('queued', 'processing', 'ready', 'failed')", name="status"),
CheckConstraint("(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", name="error_code"),
Index("ix_attachments_chat_id", "chat_id"),
)
@@ -29,7 +31,12 @@ class Attachment(Base):
content_type: Mapped[str] = mapped_column(String(32))
size_bytes: Mapped[int]
sha256: Mapped[str] = mapped_column(String(64))
status: Mapped[str] = mapped_column(String(16), server_default="ready")
status: Mapped[str] = mapped_column(String(16), server_default="queued")
error_code: Mapped[str | None] = mapped_column(String(64))
thumbnail_key: Mapped[str | None] = mapped_column(String(128))
crop_key: Mapped[str | None] = mapped_column(String(128))
optimized_key: Mapped[str | None] = mapped_column(String(128))
variants: Mapped[list[dict]] = mapped_column(JSONB, default=list, server_default=text("'[]'::jsonb"))
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
@@ -47,7 +54,7 @@ class MessageAttachment(Base):
class UploadIntent(Base):
"""Фиксируется до PUT; удаляется атомарно с публикацией ready-метаданных."""
"""Фиксируется до PUT; удаляется атомарно с принятием queued metadata/job."""
__tablename__ = "upload_intents"
__table_args__ = (Index("ix_upload_intents_created_at", "created_at"),)
+34 -13
View File
@@ -1,16 +1,37 @@
"""Место для будущих ORM-моделей job, outbox и вариантов lab5.
"""Устойчивая работа и намерение публикации; один pipeline на вложение."""
Пока моделей и регистрации в Base.metadata нет, чтобы не менять схему lab4
до реализации миграции. Планируемые данные:
from datetime import datetime
from uuid import UUID, uuid4
* job: job_id, attachment_id, pipeline_version, attempt/max_attempts,
status, lease token/deadline, next_attempt_at, безопасный error_code, timestamps;
* outbox: event_id, job_id, payload/version, состояние публикации и её lease;
* variant: attachment_id, pipeline_version, variant, приватный object_key,
content_type, size_bytes, sha256, размеры;
* replay audit: операторское действие и связь с прежней логической job.
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func
from sqlalchemy.orm import Mapped, mapped_column
Ключ эффекта (attachment_id, pipeline_version, variant) уникален. Старый worker
не публикует metadata поверх новой версии; точная схема зависит от трека 5.
Изменение ready-only constraints attachments — отдельная новая миграция.
"""
from app.models.base import Base
class AttachmentJob(Base):
__tablename__ = "attachment_jobs"
__table_args__ = (
CheckConstraint("pipeline_version = 1", name="pipeline_version"),
CheckConstraint("attempt >= 0 AND published_attempt >= 0", name="attempt"),
Index("ix_attachment_jobs_due", "next_attempt_at"),
Index("ix_attachment_jobs_lease", "lease_until"),
)
id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4)
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey("attachments.id"), unique=True,
)
staging_key: Mapped[str | None] = mapped_column(String(128))
pipeline_version: Mapped[int] = mapped_column(server_default="1")
attempt: Mapped[int] = mapped_column(server_default="0")
cycle_start_attempt: Mapped[int] = mapped_column(server_default="0")
attempt_limit: Mapped[int | None]
last_error_permanent: Mapped[bool | None]
published_attempt: Mapped[int] = mapped_column(server_default="0")
next_attempt_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
lease_token: Mapped[UUID | None]
lease_until: Mapped[datetime | None]
last_error_code: Mapped[str | None] = mapped_column(String(64))
finished_at: Mapped[datetime | None]
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+22
View File
@@ -0,0 +1,22 @@
"""Аудит явного повторного запуска failed; request id обеспечивает идемпотентность."""
from datetime import datetime
from uuid import UUID
from sqlalchemy import ForeignKey, String, func
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class JobReplay(Base):
__tablename__ = "job_replays"
id: Mapped[UUID] = mapped_column(primary_key=True)
job_id: Mapped[UUID] = mapped_column(ForeignKey("attachment_jobs.id"), index=True)
actor: Mapped[str] = mapped_column(String(64))
reason: Mapped[str] = mapped_column(String(32))
previous_attempt: Mapped[int]
attempt_limit: Mapped[int]
previous_error_code: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+33 -5
View File
@@ -1,14 +1,25 @@
"""Публичные ready-метаданные lab4; приватный object_key не выдаётся клиенту."""
"""Публичные состояния и варианты; внутренние ключи S3 не сериализуются."""
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import Field
from pydantic import BaseModel, ConfigDict, Field, field_validator
from app.schemas.base import Base
class Variant(BaseModel):
model_config = ConfigDict(extra="forbid", from_attributes=True)
name: Literal["thumbnail", "crop", "optimized"]
content_type: Literal["image/jpeg", "image/png", "image/webp"]
size_bytes: int = Field(ge=1)
width: int = Field(ge=1)
height: int = Field(ge=1)
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
class Attachment(Base):
chat_id: UUID
owner_id: UUID
@@ -16,8 +27,25 @@ class Attachment(Base):
content_type: Literal["image/png", "image/jpeg"]
size_bytes: int = Field(ge=1, le=10_485_760)
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
status: Literal["ready"] = "ready"
variants: list[object] = Field(default_factory=list, max_length=0)
error_code: None = None
status: Literal["queued", "processing", "ready", "failed"]
variants: list[Variant] = Field(default_factory=list)
error_code: str | None = None
created_at: datetime
updated_at: datetime
VARIANT_FILENAMES = {
"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp",
}
class PipelineResult(BaseModel):
model_config = ConfigDict(extra="forbid")
variants: list[Variant] = Field(min_length=3, max_length=3)
@field_validator("variants")
@classmethod
def require_all_variants(cls, variants: list[Variant]) -> list[Variant]:
if {item.name for item in variants} != set(VARIANT_FILENAMES):
raise ValueError("Нужны thumbnail, crop и optimized без повторов")
return variants
+19 -9
View File
@@ -1,10 +1,20 @@
"""Место для будущей схемы сообщения contracts/jobs.schema.json.
"""Сообщение брокера: только идентификаторы и служебные поля контракта v1."""
Пока классы не реализованы и сообщения не принимаются/не публикуются.
Обязательны schema_version=1, type=attachment.process.v1, job_id, attachment_id,
pipeline_version>=1, attempt>=1, created_at (UTC/RFC3339).
Допустимы traceparent и tracestate; дополнительные поля запрещены.
job_id сохраняется при redelivery; очередная управляемая попытка отражается
в attempt. Bytes, secrets, S3 keys и пользовательские пути не передаются.
Pydantic-схема должна соответствовать публичному JSON Schema, не менять его.
"""
from typing import Literal
from uuid import UUID
from pydantic import AwareDatetime, BaseModel, ConfigDict, Field
class JobMessage(BaseModel):
model_config = ConfigDict(extra="forbid")
schema_version: Literal[1] = 1
type: Literal["attachment.process.v1"] = "attachment.process.v1"
job_id: UUID
attachment_id: UUID
pipeline_version: int = Field(default=1, ge=1)
attempt: int = Field(ge=1)
created_at: AwareDatetime
traceparent: str | None = Field(default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
tracestate: str | None = Field(default=None, max_length=512)
+71 -20
View File
@@ -1,27 +1,78 @@
"""Заготовки адаптера durable broker; конкретный брокер ещё не выбран.
"""RabbitMQ: durable queue, persistent messages, publisher confirms и manual ack."""
Сообщения соответствуют contracts/jobs.schema.json, содержат ID и служебные
поля, но не bytes/credentials/произвольные URL. Подключение, consumer identity,
формат delivery handle и интерфейсы уточнить после выбора брокера/версии.
Здесь появятся только SDK-вызовы; решения о retry/ready находятся в domain/.
"""
import asyncio
import logging
from dataclasses import dataclass
from collections.abc import Mapping
import aio_pika
from aio_pika.abc import AbstractIncomingMessage
from app.core.jobs import JobSettings
from app.schemas.job import JobMessage
async def publish_job(payload: Mapping[str, object]) -> str:
"""Опубликовать устойчивое сообщение и вернуть идентификатор доставки.
Дождаться подтверждения сохранения брокером; неопределённый исход разрешает
повтор с тем же job_id. Не считать публикацию обработкой изображения.
"""
raise NotImplementedError("Lab5: публикация в брокер ещё не реализована")
@dataclass(frozen=True)
class Delivery:
id: AbstractIncomingMessage
payload: bytes
async def acknowledge_job(delivery_id: str) -> None:
"""Подтвердить доставку после устойчивого эффекта либо проверенного ready-дубля.
class JobBroker:
"""Один consumer/канал на экземпляр; publisher использует отдельный экземпляр."""
Retry сначала устойчиво планируется, затем закрывается старая доставка.
Конкретный delivery handle зависит от брокера; сигнатура предварительная.
"""
raise NotImplementedError("Lab5: ack ещё не реализован")
def __init__(self, settings: JobSettings) -> None:
self.settings = settings
self.connection = self.channel = self.queue = self.iterator = None
# Transport exceptions могут содержать URL с credentials.
logging.getLogger("aio_pika").setLevel(logging.CRITICAL)
logging.getLogger("aiormq").setLevel(logging.CRITICAL)
async def ensure_ready(self) -> None:
if self.connection is None or self.connection.is_closed or self.channel is None or self.channel.is_closed:
await self.aclose()
self.connection = await aio_pika.connect(
host=self.settings.broker_host, port=self.settings.broker_port,
login=self.settings.broker_user,
password=self.settings.broker_password_file.read_text().strip(),
virtualhost=self.settings.broker_vhost, timeout=5, heartbeat=30,
)
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
await self.channel.set_qos(prefetch_count=1)
self.queue = await self.channel.declare_queue(self.settings.broker_queue, durable=True)
async def publish_job(self, message: JobMessage) -> None:
await self.ensure_ready()
await self.channel.default_exchange.publish(
aio_pika.Message(
body=message.model_dump_json(exclude_none=True).encode(),
content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
message_id=f"{message.job_id}:{message.attempt}",
), routing_key=self.settings.broker_queue, mandatory=True, timeout=10,
)
async def receive(self, consumer: str) -> Delivery:
await self.ensure_ready()
if self.iterator is None:
self.iterator = self.queue.iterator(no_ack=False, consumer_tag=consumer)
await self.iterator.__aenter__()
message = await self.iterator.__anext__()
return Delivery(message, message.body)
async def acknowledge_job(self, delivery_id: AbstractIncomingMessage) -> None:
await delivery_id.ack()
async def defer_job(self, delivery_id: AbstractIncomingMessage) -> None:
# Живая чужая lease: не удерживать единственный prefetch slot навсегда.
await asyncio.sleep(self.settings.publisher_poll_seconds)
await delivery_id.nack(requeue=True)
async def reject_invalid(self, delivery_id: AbstractIncomingMessage) -> None:
# Failed jobs находятся в PostgreSQL; неизвестные payload не копируются в DLQ.
logging.getLogger(__name__).warning("Rejected invalid job envelope")
await delivery_id.reject(requeue=False)
async def aclose(self) -> None:
connection = self.connection
self.connection = self.channel = self.queue = self.iterator = None
if connection is not None and not connection.is_closed:
await connection.close()
+46
View File
@@ -0,0 +1,46 @@
"""Операторские выборки и журнал replay; транзакция принадлежит domain."""
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert
from app.models.attachment import Attachment
from app.models.job import AttachmentJob
from app.models.job_replay import JobReplay
async def failed_jobs(session, limit: int):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(Attachment.status == "failed")
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id).limit(limit)
)
return result.all()
async def replay_record(session, request_id):
return await session.get(JobReplay, request_id)
async def insert_replay(session, **values) -> bool:
result = await session.scalar(insert(JobReplay).values(**values).on_conflict_do_nothing(index_elements=[JobReplay.id]).returning(JobReplay.id))
return result is not None
async def replay_history(session, job_id):
return (await session.scalars(select(JobReplay).where(JobReplay.job_id == job_id).order_by(JobReplay.created_at).limit(100))).all()
def reset_failed(job, attachment, now, attempt_limit):
job.cycle_start_attempt = job.attempt
job.attempt_limit = attempt_limit
job.published_attempt = job.attempt
job.next_attempt_at = now
job.lease_token = None
job.lease_until = None
job.finished_at = None
job.last_error_code = None
job.last_error_permanent = None
attachment.status = "queued"
attachment.error_code = None
attachment.updated_at = now
+116 -26
View File
@@ -1,37 +1,127 @@
"""Заготовки PostgreSQL-хранилища jobs lab5; скрытых commit здесь нет.
Будущие ORM-операции захвата, lease, retry и результата должны защищаться
от конкурирующих попыток. Границы транзакций находятся в domain/.
Тип результата захвата уточняется после определения модели job.
"""
"""SQL и изменения jobs/attachments; commit остаётся в domain."""
from dataclasses import dataclass
from datetime import datetime, timedelta
from uuid import UUID
from sqlalchemy import and_, or_, func, select
from sqlalchemy.ext.asyncio import AsyncSession
async def claim_job(session: AsyncSession, job_id: UUID) -> None:
"""Условно захватить допустимое задание и сохранить lease/номер попытки.
Ready и актуальная чужая аренда не переходят обратно в processing.
Будущий результат должен различать захват, завершённый дубль и занятость.
"""
raise NotImplementedError("Lab5: захват job ещё не реализован")
from app.models.attachment import Attachment
from app.models.job import AttachmentJob
from app.schemas.attachment import VARIANT_FILENAMES
async def complete_job(session: AsyncSession, job_id: UUID) -> None:
"""Атомарно записать metadata вариантов и ready после сохранения объектов.
@dataclass(frozen=True)
class JobWork:
job_id: UUID
attachment_id: UUID
lease_token: UUID
pipeline_version: int
staging_key: str
content_type: str
size_bytes: int
sha256: str
Предусловия lease/attempt/pipeline_version и структура результата будут
добавлены в сигнатуру вместе с ORM-моделью. Не делать commit в storage.
"""
raise NotImplementedError("Lab5: фиксация результата ещё не реализована")
@property
def original_key(self) -> str:
return f"originals/{self.attachment_id.hex}/v{self.pipeline_version}"
def variant_key(self, name: str) -> str:
filename = VARIANT_FILENAMES[name]
return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}"
async def record_job_failure(session: AsyncSession, job_id: UUID, error_code: str) -> None:
"""Записать безопасный код сбоя без stack trace и secrets.
async def create_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
session.add(job)
await session.flush()
return job.id
Domain выбирает queued с next_attempt_at либо конечный failed согласно
классификации ошибки и лимиту попыток; storage лишь сохраняет решение.
"""
raise NotImplementedError("Lab5: сохранение сбоя ещё не реализовано")
async def lock_job(session: AsyncSession, job_id: UUID):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(AttachmentJob.id == job_id)
.with_for_update(of=AttachmentJob)
.execution_options(populate_existing=True)
)
return result.one_or_none()
async def database_time(session: AsyncSession) -> datetime:
return await session.scalar(select(func.clock_timestamp()))
async def expired_jobs(session: AsyncSession, limit: int = 20):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(Attachment.status == "processing", AttachmentJob.lease_until < func.clock_timestamp())
.order_by(AttachmentJob.lease_until)
.limit(limit)
.with_for_update(of=AttachmentJob, skip_locked=True)
)
return result.all()
def start_job(job, attachment, token: UUID, now: datetime, lease_seconds: int) -> JobWork:
job.attempt += 1
job.lease_token = token
job.lease_until = now + timedelta(seconds=lease_seconds)
attachment.status = "processing"
attachment.updated_at = now
return JobWork(
job.id, attachment.id, token, job.pipeline_version, job.staging_key,
attachment.content_type, attachment.size_bytes, attachment.sha256,
)
def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: bool, retry_seconds: float) -> None:
job.lease_token = None
job.lease_until = None
job.last_error_code = error_code
job.next_attempt_at = now + timedelta(seconds=retry_seconds)
job.finished_at = now if terminal else None
attachment.status = "failed" if terminal else "queued"
attachment.error_code = error_code if terminal else None
attachment.updated_at = now
def complete_job(job, attachment, work: JobWork, variants: list[dict], now: datetime) -> None:
attachment.object_key = work.original_key
attachment.thumbnail_key = work.variant_key("thumbnail")
attachment.crop_key = work.variant_key("crop")
attachment.optimized_key = work.variant_key("optimized")
attachment.variants = variants
attachment.status = "ready"
attachment.error_code = None
attachment.updated_at = now
job.lease_token = None
job.lease_until = None
job.last_error_code = None
job.last_error_permanent = None
job.finished_at = now
async def terminal_staging(session: AsyncSession, retention: int, failed_retention: int, limit: int = 20):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(
AttachmentJob.staging_key.is_not(None),
or_(
and_(Attachment.status == "ready", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=retention)),
and_(Attachment.status == "failed", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=failed_retention)),
),
)
.order_by(AttachmentJob.finished_at)
.limit(limit)
.with_for_update(of=AttachmentJob, skip_locked=True)
)
return result.all()
def clear_staging(job) -> None:
job.staging_key = None
+23 -15
View File
@@ -1,23 +1,31 @@
"""Заготовки transactional outbox для разрыва PostgreSQL COMMIT → broker.
Outbox — таблица намерений публикации в той же транзакции, что и job.
Повторная публикация после неопределённого confirm допустима; job_id стабилен.
Форма записей, захват пачки и lease publisher уточняются вместе с моделью.
"""
"""Jobs одновременно хранят намерение публикации очередной попытки."""
from uuid import UUID
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
async def add_job_event(session: AsyncSession, job_id: UUID) -> None:
"""Сохранить намерение публикации job без commit и обращения к брокеру."""
raise NotImplementedError("Lab5: запись outbox ещё не реализована")
from app.models.attachment import Attachment
from app.models.job import AttachmentJob
async def mark_event_published(session: AsyncSession, event_id: UUID) -> None:
"""Отметить событие опубликованным после устойчивого подтверждения брокера.
async def next_unpublished_job(session: AsyncSession, *, attachment_id: UUID | None = None) -> AttachmentJob | None:
statement = (
select(AttachmentJob)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(
Attachment.status == "queued",
AttachmentJob.next_attempt_at <= func.clock_timestamp(),
AttachmentJob.published_attempt <= AttachmentJob.attempt,
)
.order_by(AttachmentJob.next_attempt_at, AttachmentJob.id)
.limit(1)
.with_for_update(of=AttachmentJob, skip_locked=True)
)
if attachment_id is not None:
statement = statement.where(AttachmentJob.attachment_id == attachment_id)
return await session.scalar(statement)
Не помечать job завершённой: publisher confirm отличается от consumer ack.
"""
raise NotImplementedError("Lab5: отметка outbox ещё не реализована")
def mark_published(job: AttachmentJob) -> None:
job.published_attempt = job.attempt + 1
+121
View File
@@ -45,6 +45,28 @@ x-s3-env: &s3-env
S3_CLEANUP_GRACE_SECONDS: ${S3_CLEANUP_GRACE_SECONDS:?Set S3_CLEANUP_GRACE_SECONDS in .env}
S3_CLEANUP_INTERVAL_SECONDS: ${S3_CLEANUP_INTERVAL_SECONDS:?Set S3_CLEANUP_INTERVAL_SECONDS in .env}
x-jobs-env: &jobs-env
BROKER_HOST: ${BROKER_HOST:?Set BROKER_HOST in .env}
BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env}
BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
BROKER_QUEUE: ${BROKER_QUEUE:?Set BROKER_QUEUE in .env}
BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env}
BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:?Set WORKER_CONCURRENCY in .env}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:?Set JOB_MAX_ATTEMPTS in .env}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:?Set JOB_TIMEOUT_SECONDS in .env}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:?Set JOB_LEASE_SECONDS in .env}
JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:?Set JOB_RETRY_MAX_SECONDS in .env}
JOB_RETRY_SECONDS: ${JOB_RETRY_SECONDS:?Set JOB_RETRY_SECONDS in .env}
PUBLISHER_POLL_SECONDS: ${PUBLISHER_POLL_SECONDS:?Set PUBLISHER_POLL_SECONDS in .env}
FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env}
IMAGE_JPEG_QUALITY: ${IMAGE_JPEG_QUALITY:?Set IMAGE_JPEG_QUALITY in .env}
IMAGE_WEBP_QUALITY: ${IMAGE_WEBP_QUALITY:?Set IMAGE_WEBP_QUALITY in .env}
IMAGE_WEBP_METHOD: ${IMAGE_WEBP_METHOD:?Set IMAGE_WEBP_METHOD in .env}
IMAGE_MEMORY_MIB: ${IMAGE_MEMORY_MIB:?Set IMAGE_MEMORY_MIB in .env}
IMAGE_CPU_SECONDS: ${IMAGE_CPU_SECONDS:?Set IMAGE_CPU_SECONDS in .env}
services:
minio:
build:
@@ -154,6 +176,100 @@ services:
cpus: "0.5"
memory: 512M
broker:
image: rabbitmq:4.4.0
hostname: broker
restart: unless-stopped
entrypoint: ["/bin/sh", "/usr/local/bin/start-broker.sh"]
environment:
BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env}
BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env}
BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env}
BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
secrets:
- source: broker_password
target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
volumes:
- ./infra/broker/start-broker.sh:/usr/local/bin/start-broker.sh:ro
- rabbitmqdata:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 5s
timeout: 5s
retries: 20
start_period: 20s
deploy:
resources:
limits:
cpus: "1.0"
memory: 768M
publisher:
<<: *api-image
init: true
command: ["python", "-m", "app.core.outbox_publisher"]
restart: unless-stopped
environment:
<<: [*base-env, *jobs-env]
secrets:
- db_password
- source: broker_password
target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
depends_on:
postgres:
condition: service_healthy
broker:
condition: service_healthy
migrate:
condition: service_completed_successfully
healthcheck:
disable: true
stop_grace_period: 15s
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
worker:
<<: *api-image
init: true
command: ["python", "-m", "app.core.job_worker"]
restart: unless-stopped
environment:
<<: [*base-env, *s3-env, *jobs-env]
secrets:
- db_password
- source: broker_password
target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
- source: s3_access_key_id
target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
- source: s3_secret_access_key
target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
networks: [default, objects]
tmpfs:
- /tmp:rw,noexec,nosuid,size=128m,mode=1777
depends_on:
postgres:
condition: service_healthy
broker:
condition: service_healthy
minio:
condition: service_healthy
minio-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
healthcheck:
disable: true
stop_signal: SIGTERM
stop_grace_period: 20s
deploy:
resources:
limits:
cpus: "2.0"
memory: 2048M
migrate:
<<: *api-image
command: ["alembic", "upgrade", "head"]
@@ -229,6 +345,8 @@ services:
restart: unless-stopped
environment:
<<: [*base-env, *s3-env]
FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env}
secrets:
- db_password
- source: s3_access_key_id
@@ -274,6 +392,7 @@ volumes:
pgdata:
redisdata:
miniodata:
rabbitmqdata:
networks:
default:
@@ -281,6 +400,8 @@ networks:
internal: true
secrets:
broker_password:
file: ./secrets/broker_password
db_password:
file: ./secrets/db_password
redis_password:
+378
View File
@@ -0,0 +1,378 @@
{
"status": "passed",
"results": [
{
"scenario": "worker_stopped_restart_all",
"attachment_id": "96d4fc78-4e72-473c-a2a9-57db94421707",
"queue_before": {
"name": "attachment-jobs",
"messages_unacknowledged": 0,
"consumers": 0,
"durable": true,
"messages_ready": 1
},
"queue_after": {
"name": "attachment-jobs",
"messages_unacknowledged": 0,
"consumers": 0,
"durable": true,
"messages_ready": 1
},
"after": {
"job_id": "b3a81ec6-65d6-4201-aeb9-fedbe92fdaaf",
"status": "ready",
"attempt": 1,
"published_attempt": 1,
"limit": 5,
"updated_at": "2026-11-10 03:27:44.556734+00:00",
"next_attempt_at": "2026-11-10 03:26:56.662360+00:00",
"last_error_code": null,
"permanent": null,
"objects": [
{
"name": "crop.jpg",
"size": 2462,
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
"last_modified": "2026-11-10 03:27:44.548000+00:00"
},
{
"name": "optimized.webp",
"size": 2462,
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
"last_modified": "2026-11-10 03:27:44.552000+00:00"
},
{
"name": "thumbnail.jpg",
"size": 5813,
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
"last_modified": "2026-11-10 03:27:44.543000+00:00"
},
{
"name": "v1",
"size": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"last_modified": "2026-11-10 03:27:44.536000+00:00"
}
]
}
},
{
"scenario": "crash-after-object",
"attachment_id": "115c5e05-3d09-4f04-9b32-0d27597b65fc",
"exit_code": 137,
"before": {
"job_id": "ba3dc221-5651-4c1d-8bfd-9fd61c8aa854",
"status": "processing",
"attempt": 1,
"published_attempt": 1,
"limit": 5,
"updated_at": "2026-11-10 03:27:49.144648+00:00",
"next_attempt_at": "2026-11-10 03:27:47.218612+00:00",
"last_error_code": null,
"permanent": null,
"objects": [
{
"name": "crop.jpg",
"size": 2462,
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
"last_modified": "2026-11-10 03:27:49.912000+00:00"
},
{
"name": "optimized.webp",
"size": 2462,
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
"last_modified": "2026-11-10 03:27:49.917000+00:00"
},
{
"name": "thumbnail.jpg",
"size": 5813,
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
"last_modified": "2026-11-10 03:27:49.907000+00:00"
},
{
"name": "v1",
"size": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"last_modified": "2026-11-10 03:27:49.894000+00:00"
}
]
},
"after": {
"job_id": "ba3dc221-5651-4c1d-8bfd-9fd61c8aa854",
"status": "ready",
"attempt": 2,
"published_attempt": 2,
"limit": 5,
"updated_at": "2026-11-10 03:29:24.765539+00:00",
"next_attempt_at": "2026-11-10 03:29:23.142176+00:00",
"last_error_code": null,
"permanent": null,
"objects": [
{
"name": "crop.jpg",
"size": 2462,
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
"last_modified": "2026-11-10 03:29:24.750000+00:00"
},
{
"name": "optimized.webp",
"size": 2462,
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
"last_modified": "2026-11-10 03:29:24.758000+00:00"
},
{
"name": "thumbnail.jpg",
"size": 5813,
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
"last_modified": "2026-11-10 03:29:24.742000+00:00"
},
{
"name": "v1",
"size": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"last_modified": "2026-11-10 03:29:24.725000+00:00"
}
]
}
},
{
"scenario": "crash-before-ack",
"attachment_id": "2f97fc61-fc07-47da-98a8-9d9894ed4cd1",
"exit_code": 137,
"before": {
"job_id": "4a0bc78c-d79a-4e1a-88f7-11ad03045135",
"status": "ready",
"attempt": 1,
"published_attempt": 1,
"limit": 5,
"updated_at": "2026-11-10 03:29:29.833142+00:00",
"next_attempt_at": "2026-11-10 03:29:27.262932+00:00",
"last_error_code": null,
"permanent": null,
"objects": [
{
"name": "crop.jpg",
"size": 2462,
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
"last_modified": "2026-11-10 03:29:29.824000+00:00"
},
{
"name": "optimized.webp",
"size": 2462,
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
"last_modified": "2026-11-10 03:29:29.829000+00:00"
},
{
"name": "thumbnail.jpg",
"size": 5813,
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
"last_modified": "2026-11-10 03:29:29.818000+00:00"
},
{
"name": "v1",
"size": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"last_modified": "2026-11-10 03:29:29.805000+00:00"
}
]
},
"after": {
"job_id": "4a0bc78c-d79a-4e1a-88f7-11ad03045135",
"status": "ready",
"attempt": 1,
"published_attempt": 1,
"limit": 5,
"updated_at": "2026-11-10 03:29:29.833142+00:00",
"next_attempt_at": "2026-11-10 03:29:27.262932+00:00",
"last_error_code": null,
"permanent": null,
"objects": [
{
"name": "crop.jpg",
"size": 2462,
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
"last_modified": "2026-11-10 03:29:29.824000+00:00"
},
{
"name": "optimized.webp",
"size": 2462,
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
"last_modified": "2026-11-10 03:29:29.829000+00:00"
},
{
"name": "thumbnail.jpg",
"size": 5813,
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
"last_modified": "2026-11-10 03:29:29.818000+00:00"
},
{
"name": "v1",
"size": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"last_modified": "2026-11-10 03:29:29.805000+00:00"
}
]
}
},
{
"scenario": "fail-transient",
"attachment_id": "92c4de14-53cd-4805-a842-9892a8b7a5ea",
"samples": [
{
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
"status": "queued",
"attempt": 1,
"published_attempt": 1,
"limit": 5,
"updated_at": "2026-11-10 03:29:45.863494+00:00",
"next_attempt_at": "2026-11-10 03:29:50.476742+00:00",
"last_error_code": "processing_timeout",
"permanent": false,
"objects": []
},
{
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
"status": "queued",
"attempt": 2,
"published_attempt": 2,
"limit": 5,
"updated_at": "2026-11-10 03:29:51.216904+00:00",
"next_attempt_at": "2026-11-10 03:29:58.053724+00:00",
"last_error_code": "processing_timeout",
"permanent": false,
"objects": []
},
{
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
"status": "queued",
"attempt": 3,
"published_attempt": 3,
"limit": 5,
"updated_at": "2026-11-10 03:29:58.279470+00:00",
"next_attempt_at": "2026-11-10 03:30:15.602067+00:00",
"last_error_code": "processing_timeout",
"permanent": false,
"objects": []
},
{
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
"status": "queued",
"attempt": 4,
"published_attempt": 4,
"limit": 5,
"updated_at": "2026-11-10 03:30:16.386336+00:00",
"next_attempt_at": "2026-11-10 03:30:47.271081+00:00",
"last_error_code": "processing_timeout",
"permanent": false,
"objects": []
},
{
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
"status": "failed",
"attempt": 5,
"published_attempt": 5,
"limit": 5,
"updated_at": "2026-11-10 03:30:47.622716+00:00",
"next_attempt_at": "2026-11-10 03:31:30.255216+00:00",
"last_error_code": "processing_timeout",
"permanent": false,
"objects": []
}
],
"after": {
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
"status": "ready",
"attempt": 6,
"published_attempt": 6,
"limit": 10,
"updated_at": "2026-11-10 03:30:59.696883+00:00",
"next_attempt_at": "2026-11-10 03:30:53.135190+00:00",
"last_error_code": null,
"permanent": null,
"objects": [
{
"name": "crop.jpg",
"size": 2462,
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
"last_modified": "2026-11-10 03:30:59.687000+00:00"
},
{
"name": "optimized.webp",
"size": 2462,
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
"last_modified": "2026-11-10 03:30:59.692000+00:00"
},
{
"name": "thumbnail.jpg",
"size": 5813,
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
"last_modified": "2026-11-10 03:30:59.683000+00:00"
},
{
"name": "v1",
"size": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"last_modified": "2026-11-10 03:30:59.670000+00:00"
}
]
},
"replay_idempotent": true
},
{
"scenario": "fail-permanent",
"attachment_id": "92e2baff-5b26-4326-bee6-269382ff80c9",
"samples": [
{
"job_id": "f462098d-43b7-41cc-afe8-399aaf3ed0bd",
"status": "failed",
"attempt": 1,
"published_attempt": 1,
"limit": 5,
"updated_at": "2026-11-10 03:31:12.333162+00:00",
"next_attempt_at": "2026-11-10 03:31:16.827024+00:00",
"last_error_code": "injected_permanent",
"permanent": true,
"objects": []
}
],
"after": {
"job_id": "f462098d-43b7-41cc-afe8-399aaf3ed0bd",
"status": "ready",
"attempt": 2,
"published_attempt": 2,
"limit": 6,
"updated_at": "2026-11-10 03:31:24.282794+00:00",
"next_attempt_at": "2026-11-10 03:31:17.137221+00:00",
"last_error_code": null,
"permanent": null,
"objects": [
{
"name": "crop.jpg",
"size": 2462,
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
"last_modified": "2026-11-10 03:31:24.272000+00:00"
},
{
"name": "optimized.webp",
"size": 2462,
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
"last_modified": "2026-11-10 03:31:24.278000+00:00"
},
{
"name": "thumbnail.jpg",
"size": 5813,
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
"last_modified": "2026-11-10 03:31:24.267000+00:00"
},
{
"name": "v1",
"size": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"last_modified": "2026-11-10 03:31:24.254000+00:00"
}
]
},
"replay_idempotent": true
}
]
}
+271
View File
@@ -0,0 +1,271 @@
{
"date": "2026-11-10",
"status": "passed",
"project": "mtusi-lab5-verification",
"base_url": "http://localhost:28080",
"checks": [
{
"name": "pytest_regression",
"exit_code": 0,
"stdout": "199 passed in 82.29s (0:01:22)",
"scope": "Dedicated *_test database and Redis DB15; Docker restart tests enabled"
},
{
"name": "pytest_timeout_retention",
"exit_code": 0,
"stdout": "2 passed, 5 deselected in 1.91s",
"scope": "Dedicated *_test database and Redis DB15; Docker restart tests enabled"
},
{
"name": "compose_configuration",
"command": [
"docker",
"compose",
"--env-file",
".env",
"config",
"--quiet"
],
"exit_code": 0,
"stdout": ""
},
{
"name": "contract",
"command": [
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
"scripts/check_contract.py"
],
"exit_code": 0,
"stdout": "PASS: offline skeleton structure, local refs and Python syntax; application NOT tested."
},
{
"name": "openapi",
"command": [
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
"scripts/validate_openapi.py"
],
"exit_code": 0,
"stdout": "PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested."
},
{
"name": "public_smoke",
"command": [
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
"tests/smoke.py",
"--base-url",
"http://localhost:28080"
],
"exit_code": 0,
"stdout": "PASS: lab5 public API scenario; infrastructure and grading evidence remain separate."
},
{
"name": "message_idempotency",
"command": [
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
"-m",
"scripts.check_idempotency",
"--base-url",
"http://localhost:28080"
],
"exit_code": 0,
"stdout": "PASS: 24 concurrent replays, 2 replicas, one message"
},
{
"name": "outbox_crash",
"command": [
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
"-m",
"scripts.check_outbox_recovery",
"--base-url",
"http://localhost:28080"
],
"exit_code": 0,
"stdout": "{\"scenario\": \"commit_before_publish\", \"attachment_id\": \"0fff1dce-c479-40d8-8627-f28cc85cb6cf\", \"message_id\": \"e79bfb7e-c2af-4011-a87a-52528d7a7840\", \"publisher_exit_code\": 137, \"status_after_crash\": \"queued\", \"status_after_restart\": \"ready\", \"variants\": [\"thumbnail\", \"crop\", \"optimized\"], \"reuploaded\": false}"
},
{
"name": "public_persistence",
"command": [
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
"tests/resilience.py",
"--mode",
"persistence",
"--action-script",
"scripts/restart.sh",
"--base-url",
"http://localhost:28080"
],
"exit_code": 0,
"stdout": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence."
},
{
"name": "public_failover",
"command": [
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
"tests/resilience.py",
"--mode",
"failover",
"--action-script",
"scripts/stop-one.sh",
"--base-url",
"http://localhost:28080"
],
"exit_code": 0,
"stdout": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 104, \"failed_attempts\": 5, \"first_recovery_seconds\": 1.907}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence."
},
{
"name": "rabbitmq_prefetch",
"command": [
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"broker",
"rabbitmqctl",
"list_channels",
"--formatter=json",
"consumer_count",
"prefetch_count",
"messages_unacknowledged"
],
"exit_code": 0,
"stdout": "[\n{\"consumer_count\":1,\"messages_unacknowledged\":0,\"prefetch_count\":1}\n,{\"consumer_count\":0,\"messages_unacknowledged\":0,\"prefetch_count\":1}\n,{\"consumer_count\":1,\"messages_unacknowledged\":0,\"prefetch_count\":1}\n]"
},
{
"name": "rabbitmq_durability_ack",
"command": [
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"broker",
"rabbitmqctl",
"list_queues",
"--formatter=json",
"name",
"durable",
"messages_ready",
"messages_unacknowledged",
"consumers"
],
"exit_code": 0,
"stdout": "[\n{\"name\":\"attachment-jobs\",\"messages_unacknowledged\":0,\"consumers\":2,\"durable\":true,\"messages_ready\":0}\n]"
},
{
"name": "worker_fault_retry_replay",
"exit_code": 0,
"evidence": "lab5-jobs.json",
"scenarios": [
"worker_stopped_restart_all",
"crash-after-object",
"crash-before-ack",
"fail-transient",
"fail-permanent"
]
},
{
"name": "whitespace",
"command": [
"git",
"diff",
"--check"
],
"exit_code": 0,
"stdout": ""
},
{
"name": "public_contract_tests_unchanged",
"exit_code": 0
},
{
"name": "final_services",
"command": [
"docker",
"compose",
"--env-file",
".env",
"ps",
"--format",
"{{.Service}} {{.State}} {{.Health}}"
],
"exit_code": 0,
"stdout": "api running healthy\napi running healthy\nbroker running healthy\ncleanup running \nminio running healthy\nnginx running healthy\npostgres running healthy\npublisher running \nredis running healthy\nworker running"
}
],
"git_base_commit": "0a703588290e26d0f148b45d0e6bee2e8eefce7d",
"uncommitted_changes": true,
"environment": {
"test_runner_python": "3.12.14",
"runtime_python": "3.12.15",
"broker": "RabbitMQ 4.4.0",
"sessions": "Redis 7.4.2",
"worker_memory_bytes": 2147483648,
"worker_cpu_limit": 2.0,
"worker_init": true
},
"profile": {
"data": "Synthetic users/chats only; tests/fixtures/sample.png for fault scenarios; no credentials in evidence",
"worker_concurrency": 2,
"prefetch_per_consumer": 1,
"max_attempts_per_cycle": 5,
"timeout_seconds": 60,
"lease_seconds": 90,
"retry_base_seconds": 5,
"retry_cap_seconds": 120,
"image_memory_mib": 768,
"image_cpu_seconds": 20,
"isolated_limit_tests": "256 MiB / 1 CPU second in child process; timeout cancellation checked separately"
},
"criteria": {
"3.1_lab4_and_durable_202": [
"pytest_regression",
"public_smoke",
"public_persistence",
"worker_fault_retry_replay"
],
"3.2_durable_broker_worker_late_ack": [
"rabbitmq_durability_ack",
"rabbitmq_prefetch",
"worker_fault_retry_replay"
],
"3.3_states_original_thumbnail": [
"public_smoke",
"pytest_regression",
"worker_fault_retry_replay"
],
"3.4_bounded_retry_idempotency": [
"pytest_regression",
"worker_fault_retry_replay"
],
"3.5_stopped_worker_api_broker_restart": [
"worker_fault_retry_replay",
"public_persistence"
],
"4.1_crop_webp_and_worker_limits": [
"pytest_regression",
"worker_fault_retry_replay"
],
"4.2_commit_publish_recovery": [
"outbox_crash"
],
"4.3_backoff_jitter_classification_registry_replay": [
"pytest_regression",
"pytest_timeout_retention",
"worker_fault_retry_replay"
],
"4.4_crash_after_put_lease_prefetch_cleanup": [
"worker_fault_retry_replay",
"rabbitmq_prefetch",
"pytest_regression",
"pytest_timeout_retention"
]
},
"limitations": [
"Single Docker host, no disk-loss or cluster failover claim",
"Grade 5 experiments not performed",
"No production-scale performance/security claim"
]
}
+21 -11
View File
@@ -1,17 +1,27 @@
# Инфраструктура студента
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
Топология — compose.yaml. compose.test.yaml публикует тестовые PostgreSQL/Redis
только на localhost. Nginx направляет запросы на две API-реплики.
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
PostgreSQL хранит сообщения, metadata, jobs/outbox и аудит replay. Redis 7.4.2
с AOF хранит сессии. RabbitMQ 4.4.0 — отдельный брокер заданий: durable очередь,
persistent сообщения, publisher confirms, manual consumer ack и rabbitmqdata.
Broker и MinIO не публикуют внешние порты. API не получает credentials брокера.
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
Publisher читает jobs из PostgreSQL и публикует их в RabbitMQ. Worker сохраняет
original/thumbnail/crop/optimized в приватный MinIO и фиксирует ready до ack.
Конкуренция ограничена WORKER_CONCURRENCY; prefetch=1 на отдельный канал каждого
consumer. Publisher/worker используют init:true для корректных сигналов и
воспроизводимого SIGKILL. Обычный Compose-запуск fault hooks не включает.
## Перенесённая база lab4
Все environment-настройки Compose берутся из .env; образец — .env.example.
Credentials передаются файлами secrets; make init создаёт только отсутствующие.
Migrate и minio-bootstrap завершаются до старта зависимых сервисов.
Перед обновлением остановить старые процессы без удаления volumes. Миграция
a62c950e18b3 повторно публикует queued jobs при переходе на RabbitMQ.
В compose.yaml перенесены postgres, redis (сессии), minio, minio-bootstrap,
migrate, api (две реплики), nginx и cleanup. Версии сохранены без обновления.
Роли broker, outbox publisher и worker пока не реализованы.
core/image_worker.py — одноразовая проверка изображения lab4, а не consumer очереди.
Имена проектов/образов lab5 отделены от lab4. Запуск и порты — в REPORT.md.
Настройки environment в Compose задаются через .env; образец — .env.example.
Секретные значения монтируются отдельно, пути *_FILE относятся к контейнерам.
Restart hook сохраняет volumes и состояние заранее остановленного worker.
Failover hook останавливает выбранную API-реплику. Retry, retention, операторские
команды, ограничения и результаты проверок описаны в REPORT.md.
Полный прогон lab5 уровня 4 — make test-grade4; отдельные сценарии —
make test-outbox-recovery и make test-jobs на выбранном работающем стенде.
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
# Secret читается внутри контейнера, не попадает в compose config/аргументы.
export RABBITMQ_DEFAULT_PASS="$(cat "${BROKER_PASSWORD_FILE:?}")"
export RABBITMQ_DEFAULT_USER="${BROKER_USER:?}"
export RABBITMQ_DEFAULT_VHOST="${BROKER_VHOST:?}"
export RABBITMQ_NODE_PORT="${BROKER_PORT:?}"
exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server
+1 -1
View File
@@ -81,7 +81,7 @@ def main():
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": [f"arn:aws:s3:::{bucket}/originals/*"],
"Resource": [f"arn:aws:s3:::{bucket}/{prefix}/*" for prefix in ("originals", "staging", "variants")],
},
],
}
@@ -0,0 +1,21 @@
"""Offline cutover: повторно опубликовать queued jobs в RabbitMQ."""
from alembic import op
revision = "a62c950e18b3"
down_revision = "f9183d27a640"
branch_labels = None
depends_on = None
def upgrade() -> None:
# Старые publisher/worker остановлены. Processing восстановится по lease.
op.execute("""
UPDATE attachment_jobs j SET published_attempt = attempt
FROM attachments a WHERE a.id = j.attachment_id AND a.status = 'queued'
""")
def downgrade() -> None:
# При обратном переключении требуется такая же повторная публикация.
upgrade()
@@ -0,0 +1,62 @@
"""Lab5: состояния, варианты и устойчивые jobs с намерением публикации."""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision = "c35e91a2f706"
down_revision = "8f31b94a2d10"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check")
op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status IN ('queued', 'processing', 'ready', 'failed')")
op.alter_column("attachments", "status", server_default="queued")
op.add_column("attachments", sa.Column("error_code", sa.String(64), nullable=True))
op.add_column("attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True))
op.add_column("attachments", sa.Column("variants", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb")))
op.create_check_constraint(op.f("ck_attachments_error_code"), "attachments", "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)")
op.create_table(
"attachment_jobs",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("attachment_id", sa.UUID(), nullable=False),
sa.Column("staging_key", sa.String(128), nullable=True),
sa.Column("pipeline_version", sa.Integer(), server_default="1", nullable=False),
sa.Column("attempt", sa.Integer(), server_default="0", nullable=False),
sa.Column("published_attempt", sa.Integer(), server_default="0", nullable=False),
sa.Column("next_attempt_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.Column("lease_token", sa.UUID(), nullable=True),
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error_code", sa.String(64), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.PrimaryKeyConstraint("id", name=op.f("pk_attachment_jobs")),
sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_attachment_jobs_attachment_id_attachments")),
sa.UniqueConstraint("attachment_id", name=op.f("uq_attachment_jobs_attachment_id")),
sa.CheckConstraint("pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version")),
sa.CheckConstraint("attempt >= 0 AND published_attempt >= 0", name=op.f("ck_attachment_jobs_attempt")),
)
op.create_index("ix_attachment_jobs_due", "attachment_jobs", ["next_attempt_at"])
op.create_index("ix_attachment_jobs_lease", "attachment_jobs", ["lease_until"])
# Прежние оригиналы остаются источником; worker создаст отсутствующий thumbnail.
op.execute("INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments")
op.execute("UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()")
def downgrade() -> None:
connection = op.get_bind()
unfinished = connection.scalar(sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'"))
if unfinished:
raise RuntimeError("Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения")
op.drop_index("ix_attachment_jobs_lease", table_name="attachment_jobs")
op.drop_index("ix_attachment_jobs_due", table_name="attachment_jobs")
op.drop_table("attachment_jobs")
op.drop_constraint(op.f("ck_attachments_error_code"), "attachments", type_="check")
op.drop_column("attachments", "variants")
op.drop_column("attachments", "thumbnail_key")
op.drop_column("attachments", "error_code")
op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check")
op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status = 'ready'")
op.alter_column("attachments", "status", server_default="ready")
@@ -0,0 +1,49 @@
"""Уровень 4: ключи crop/WebP и дообработка старых ready-вложений."""
from alembic import op
import sqlalchemy as sa
revision = "e7048b6da913"
down_revision = "c35e91a2f706"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("attachments", sa.Column("crop_key", sa.String(128), nullable=True))
op.add_column("attachments", sa.Column("optimized_key", sa.String(128), nullable=True))
# Старый staging сменяется надёжным ready-оригиналом, но не теряется из журнала cleanup.
op.execute("""
INSERT INTO upload_intents (id, object_key)
SELECT gen_random_uuid(), j.staging_key
FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE a.status = 'ready' AND j.staging_key IS NOT NULL
AND j.staging_key <> a.object_key
ON CONFLICT (object_key) DO NOTHING
""")
op.execute("""
UPDATE attachment_jobs j
SET staging_key = a.object_key, attempt = 0, published_attempt = 0,
next_attempt_at = clock_timestamp(), lease_token = NULL, lease_until = NULL,
last_error_code = NULL, finished_at = NULL
FROM attachments a
WHERE a.id = j.attachment_id AND a.status = 'ready'
""")
op.execute("""
UPDATE attachments SET status = 'queued', variants = '[]'::jsonb,
updated_at = clock_timestamp()
WHERE status = 'ready'
""")
def downgrade() -> None:
active = op.get_bind().scalar(sa.text("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')"))
if active:
raise RuntimeError("Перед downgrade необходимо завершить активные задания")
op.execute("""
UPDATE attachments a SET variants = COALESCE(
(SELECT jsonb_agg(v) FROM jsonb_array_elements(a.variants) v
WHERE v->>'name' = 'thumbnail'), '[]'::jsonb)
""")
op.drop_column("attachments", "optimized_key")
op.drop_column("attachments", "crop_key")
@@ -0,0 +1,34 @@
"""Бюджет повторов и аудит операторского replay."""
from alembic import op
import sqlalchemy as sa
revision = "f9183d27a640"
down_revision = "e7048b6da913"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("attachment_jobs", sa.Column("cycle_start_attempt", sa.Integer(), nullable=False, server_default="0"))
op.add_column("attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True))
op.add_column("attachment_jobs", sa.Column("last_error_permanent", sa.Boolean(), nullable=True))
op.create_table(
"job_replays",
sa.Column("id", sa.Uuid(), primary_key=True),
sa.Column("job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False),
sa.Column("actor", sa.String(64), nullable=False),
sa.Column("reason", sa.String(32), nullable=False),
sa.Column("previous_attempt", sa.Integer(), nullable=False),
sa.Column("attempt_limit", sa.Integer(), nullable=False),
sa.Column("previous_error_code", sa.String(64), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.clock_timestamp()),
)
op.create_index("ix_job_replays_job_id", "job_replays", ["job_id"])
def downgrade() -> None:
op.drop_table("job_replays")
op.drop_column("attachment_jobs", "last_error_permanent")
op.drop_column("attachment_jobs", "attempt_limit")
op.drop_column("attachment_jobs", "cycle_start_attempt")
+1
View File
@@ -50,3 +50,4 @@ pytest-asyncio==1.4.1
httpx==0.28.1
minio==7.2.20
Pillow==13.0.0
aio-pika==10.2.2
+210
View File
@@ -0,0 +1,210 @@
"""Fault/recovery lab5 на явно выбранном Compose-стенде; только синтетические данные."""
import argparse
from datetime import datetime
import hashlib
from io import BytesIO
import json
from pathlib import Path
import subprocess
import time
from uuid import UUID, uuid4
from PIL import Image
from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose
from scripts.deployment import ComposeProject
from tests.client import Client, check
PROBE = '''
import asyncio, hashlib, json
from uuid import UUID
from sqlalchemy import select
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.models.job import AttachmentJob
from app.models.attachment import Attachment
from app.storage.s3 import create_s3_storage
async def main():
settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3)
try:
async with create_session_factory(engine)() as session:
job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one()
result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent)
objects=[]
for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'):
entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True)))
for obj in entries:
stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256()
try:
async for chunk in stream: checksum.update(chunk)
finally: await stream.aclose()
objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified))
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
finally: await storage.aclose();await engine.dispose()
asyncio.run(main())
'''
def probe(attachment_id):
code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE
result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30)
check(result.returncode == 0, 'Probe failed; diagnostic output hidden')
return json.loads(result.stdout)
def admin(*arguments, expected=0):
result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments)
check(result.returncode == expected, 'Unexpected operator command exit status')
return json.loads(result.stdout)
def verify_result(client, actor, attachment, original):
ready = client.wait_ready(actor, attachment, wait_seconds=180)
check(ready['id'] == attachment['id'], 'Attachment identity changed')
path = '/api/v1/attachments/' + ready['id'] + '/content'
check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed')
check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline')
for variant in ready['variants']:
data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes']
check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch')
with Image.open(BytesIO(data)) as image:
check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch')
check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format')
check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked')
if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size')
snapshot=probe(ready['id'])
check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants')
return snapshot
def crash(client, actor, chat, mode):
name='lab5-worker-fault-'+uuid4().hex
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack'
marker=json.dumps({'fault':phase,'attachment_id':attachment['id']})
check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed')
before=probe(attachment['id'])
check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state')
check(len(before['objects'])==4,'Crash must occur after all four PUTs')
finally:
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
if mode=='crash-before-ack':
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
wait_queue_empty()
after=probe(attachment['id'])
check(after==before,'Ready duplicate rewrote result')
else:
check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt')
check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content')
return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after)
def retry_replay(client,actor,chat,permanent):
name='lab5-retry-fault-'+uuid4().hex
samples=[]
configuration=ComposeProject().config['services']['worker']['environment']
retry_base=float(configuration['JOB_RETRY_SECONDS'])
retry_max=float(configuration['JOB_RETRY_MAX_SECONDS'])
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
mode='fail-permanent' if permanent else 'fail-transient'
require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
deadline=time.monotonic()+180
while time.monotonic()<deadline:
sample=probe(attachment['id'])
if sample['attempt'] and sample['status'] in {'queued', 'failed'} and (not samples or samples[-1]['attempt']!=sample['attempt']):samples.append(sample)
if sample['status']=='failed':break
time.sleep(.5)
check(sample['status']=='failed','Retry never reached failed')
check(sample['attempt']==(1 if permanent else sample['limit']),'Incorrect attempt budget')
check(sample['permanent']==permanent,'Incorrect error classification')
if not permanent:
check(len(samples)==sample['limit'],'Missed retry samples')
for entry in samples:
delay=(datetime.fromisoformat(entry['next_attempt_at'])-datetime.fromisoformat(entry['updated_at'])).total_seconds()
ceiling=min(retry_max,retry_base*2**(entry['attempt']-1))
check(ceiling/2<=delay<=ceiling,'Retry delay outside configured jitter bounds')
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
job_id=sample['job_id'];request_id=str(uuid4())
listing=admin('list-failed')
check(any(row['job_id']==job_id for row in listing),'Job missing from failed registry')
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30,check=True)
first=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
check(first['status']=='queued','Replay did not queue job')
finally:
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
repeat=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
check(repeat['already_applied'],'Replay request id not idempotent')
refused=admin('replay',job_id,'--request-id',str(uuid4()),'--actor','verification','--reason','manual_review',expected=2)
check(refused['error_code']=='job_not_failed','Successful job was replayed')
check(probe(attachment['id'])==after,'Operator replay altered ready job')
check(len(admin('history',job_id))==1,'Incorrect audit count')
return dict(scenario=mode,attachment_id=attachment['id'],samples=samples,after=after,replay_idempotent=True)
def queue_state():
output=require_compose('exec','-T','broker','rabbitmqctl','list_queues','--formatter=json','name','messages_ready','messages_unacknowledged','consumers','durable')
return next(row for row in json.loads(output) if row['name']=='attachment-jobs')
def wait_queue_empty():
deadline=time.monotonic()+20
while time.monotonic()<deadline:
state=queue_state()
if state['messages_ready']==0 and state['messages_unacknowledged']==0:
return state
time.sleep(.5)
check(False,'RabbitMQ did not acknowledge all completed deliveries')
def persistence(client,actor,chat):
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
client.message(actor,chat,'',attachments=[attachment['id']])
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
time.sleep(2)
queued_before=queue_state()
check(queued_before['messages_ready']>=1 and queued_before['durable'], 'No durable queued message before restart')
result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180)
check(result.returncode==0,'Restart hook failed')
check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker')
queued_after=queue_state()
check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart')
client.message(actor,chat,'session and chat survived broker/API restart')
finally:require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after)
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--base-url',required=True)
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
parser.add_argument('--output',default='evidence/lab5-jobs.json')
args=parser.parse_args()
client=Client(args.base_url,timeout=40)
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
actor,_,_,chat=client.bootstrap()
results=[]
operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)}
selected=operations.values() if args.scenario=='all' else [operations[args.scenario]]
for operation in selected:
result=operation(client,actor,chat);results.append(result)
Path(args.output).parent.mkdir(parents=True,exist_ok=True)
Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n')
print('PASS: '+result['scenario'],flush=True)
Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n')
if __name__=='__main__':main()
+83
View File
@@ -0,0 +1,83 @@
"""Сценарий COMMIT → SIGKILL до publish → ready без повторного upload.
Требуется уже работающий отдельный стенд. Скрипт не выполняется при запуске API.
Публичные tests не изменяет, secrets/tokens не печатает и evidence сам не сохраняет.
"""
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
from uuid import uuid4
from scripts.deployment import ComposeProject
from tests.client import Client, check
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess:
return subprocess.run(COMPOSE + list(arguments), cwd=ROOT, text=True, capture_output=True, timeout=timeout)
def require_compose(*arguments: str) -> str:
result = compose(*arguments)
check(result.returncode == 0, "Compose command failed; output hidden")
return result.stdout
def main() -> None:
parser = argparse.ArgumentParser(description="Отказ publisher после COMMIT перед RabbitMQ publish")
parser.add_argument("--base-url", required=True)
parser.add_argument("--ca-file")
args = parser.parse_args()
running = set(require_compose("ps", "--status", "running", "--services").splitlines())
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the lab5 stack before this scenario")
client = Client(args.base_url, args.ca_file, timeout=40)
project = ComposeProject()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
live = client.call("GET", "/health/live", schema="Health")
check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project")
alice, _, _, chat = client.bootstrap()
attachment = None
fault_container = "lab5-outbox-fault-" + uuid4().hex
try:
require_compose("stop", "publisher")
attachment, original = client.attachment(alice, chat)
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected=409, actor=alice)
killed = compose(
"run", "--rm", "--no-deps", "--name", fault_container, "publisher", "python", "-m",
"app.core.outbox_publisher", "--crash-before-publish", attachment["id"],
)
marker = json.dumps({"fault": "before_publish", "attachment_id": attachment["id"]})
check(killed.returncode == 137 and marker in killed.stdout, "Expected targeted SIGKILL before publish was not observed")
pending = client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=alice, schema="Attachment")["json"]
check(pending["status"] == "queued", "Crash must leave accepted job queued")
finally:
# Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется.
try:
subprocess.run(["docker", "rm", "-f", fault_container], capture_output=True, timeout=30)
finally:
require_compose("start", "publisher")
ready = client.wait_ready(alice, attachment, wait_seconds=60)
check(ready["id"] == attachment["id"], "Recovered a different attachment")
check({variant["name"] for variant in ready["variants"]} == {"thumbnail", "crop", "optimized"}, "Missing required variants")
path = f"/api/v1/attachments/{attachment['id']}/content"
check(client.call("GET", path, actor=alice)["bytes"] == original, "Original changed after recovery")
for variant in ready["variants"]:
downloaded = client.call("GET", path + "?variant=" + variant["name"], actor=alice)["bytes"]
check(len(downloaded) == variant["size_bytes"] and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], "Recovered variant checksum mismatch")
print(json.dumps({
"scenario": "commit_before_publish", "attachment_id": ready["id"],
"message_id": message["id"], "publisher_exit_code": killed.returncode,
"status_after_crash": pending["status"], "status_after_restart": ready["status"],
"variants": [variant["name"] for variant in ready["variants"]],
"reuploaded": False,
}))
if __name__ == "__main__":
main()
+1 -1
View File
@@ -14,7 +14,7 @@ def main() -> None:
destination.chmod(0o600)
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
for name in ("db_password", "redis_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"):
for name in ("db_password", "redis_password", "broker_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"):
path = directory / name
try:
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444)
+9 -4
View File
@@ -22,11 +22,16 @@ wait_healthy() {
done
}
compose stop cleanup
compose restart postgres redis minio
wait_healthy postgres redis minio
worker_running=$(compose ps --status running --quiet worker)
compose stop worker publisher cleanup
compose restart postgres redis minio broker
wait_healthy postgres redis minio broker
compose restart api
wait_healthy api
compose restart nginx
wait_healthy nginx
compose start cleanup
compose start publisher cleanup
# Сценарий с заранее остановленным worker сохраняет его остановленным.
if [ -n "$worker_running" ]; then
compose start worker
fi
+32 -8
View File
@@ -1,4 +1,4 @@
"""Lab4: API и транзакции с реальной БД; байты S3 подменены управляемым адаптером."""
"""Lab5: API и транзакции с реальной БД; байты S3 подменены управляемым адаптером."""
import asyncio
import hashlib
@@ -17,6 +17,10 @@ from sqlalchemy.ext.asyncio import AsyncSession
from starlette.requests import ClientDisconnect
from app.core.s3 import S3Settings
from app.core.jobs import JobSettings
from app.domain.attachment_jobs import process_attachment_job
from app.models.job import AttachmentJob
from app.schemas.job import JobMessage
from app.domain import attachments as domain
from app.errors.exceptions import unavailable
from app.models.attachment import Attachment, UploadIntent
@@ -79,14 +83,31 @@ async def upload(client, actor, chat, data=PNG, filename='фото.png', content
)
async def finish_attachment(client, storage, actor, attachment_id):
engine = client._transport.app.state.engine
async with AsyncSession(engine) as session:
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
message = JobMessage(job_id=job.id, attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version, attempt=job.attempt + 1, created_at=job.created_at)
await session.rollback()
assert await process_attachment_job(session, storage, message, JobSettings(_env_file=None))
result = await client.get(f'/api/v1/attachments/{attachment_id}', headers=bearer(actor))
assert result.json()['status'] == 'ready'
return result.json()
async def test_upload_download_and_attachment_only_message(client, context):
storage, alice, bob, eve, chat = context
response = await upload(client, alice, chat)
assert response.status_code == 201, response.text
assert response.status_code == 202, response.text
attachment = response.json()
assert attachment['status'] == 'ready'
assert attachment['status'] == 'queued'
before = await client.get(f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice))
assert before.status_code == 409
attachment = await finish_attachment(client, storage, alice, attachment['id'])
assert attachment['sha256'] == hashlib.sha256(PNG).hexdigest()
assert attachment['variants'] == [] and attachment['error_code'] is None
assert {v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'}
assert attachment['error_code'] is None
assert 'object_key' not in attachment
id = attachment['id']
for actor in (alice, bob):
@@ -120,13 +141,14 @@ async def test_invalid_image_and_mime_are_415(client, context, data, content_typ
async def test_jpeg_is_stored_without_reencoding(client, context):
_, alice, _, _, chat = context
storage, alice, _, _, chat = context
output = BytesIO()
Image.new('RGB', (32, 32), 'blue').save(output, format='JPEG')
original = output.getvalue()
response = await upload(client, alice, chat, original, content_type='image/jpeg')
assert response.status_code == 201, response.text
assert response.status_code == 202, response.text
id = response.json()['id']
await finish_attachment(client, storage, alice, id)
downloaded = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
assert downloaded.content == original
@@ -167,7 +189,7 @@ async def test_exact_byte_limit_accepts_valid_png(client, context):
padding = struct.pack('!I', len(payload)) + kind + payload + struct.pack('!I', zlib.crc32(kind + payload))
image = PNG[:-12] + padding + PNG[-12:]
response = await upload(client, alice, chat, image)
assert response.status_code == 201, response.text
assert response.status_code == 202, response.text
assert response.json()['size_bytes'] == maximum
assert response.json()['sha256'] == hashlib.sha256(image).hexdigest()
@@ -178,7 +200,7 @@ async def test_exact_pixel_limit_accepts_valid_png(client, context):
with Image.new('RGB', (8000, 5000), 'blue') as image:
image.save(output, format='PNG')
response = await upload(client, alice, chat, output.getvalue())
assert response.status_code == 201, response.text
assert response.status_code == 202, response.text
async def test_pixel_limit_before_decoding(client, context):
@@ -226,6 +248,7 @@ async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context):
async def test_s3_outage_is_503(client, context):
storage, alice, _, _, chat = context
id = (await upload(client, alice, chat)).json()['id']
await finish_attachment(client, storage, alice, id)
storage.down = True
response = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
assert response.status_code == 503
@@ -353,6 +376,7 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa
async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine):
storage, alice, _, _, chat = context
id = (await upload(client, alice, chat)).json()['id']
await finish_attachment(client, storage, alice, id)
async with AsyncSession(db_engine) as session:
attachment = await session.get(Attachment, UUID(id))
key = attachment.object_key
+71
View File
@@ -0,0 +1,71 @@
"""Реальные преобразования и OS-лимиты дочернего процесса worker."""
import asyncio
from io import BytesIO
import os
from pathlib import Path
import subprocess
import sys
from uuid import uuid4
from PIL import Image
import pytest
from app.core.image_processing import ImageSettings
from app.domain.image_pipeline import create_variants
from app.domain.job_files import render_variants
from app.storage.jobs import JobWork
from test_attachments import MemoryStorage
@pytest.mark.parametrize('size,transparent', [((1600,800),False),((24,12),True)])
async def test_variant_geometry_metadata_and_alpha(tmp_path,size,transparent):
source=tmp_path/'source.png'
with Image.new('RGBA',size,(255,0,0,0 if transparent else 255)) as image:
exif=Image.Exif();exif[270]='private description'
image.save(source,format='PNG',exif=exif)
result=create_variants(source,tmp_path,'image/png',40_000_000,ImageSettings(_env_file=None))
variants={v.name:v for v in result.variants}
assert (variants['crop'].width,variants['crop'].height)==(128,128)
assert (variants['thumbnail'].width,variants['thumbnail'].height)==((256,128) if size[0]>256 else size)
assert (variants['optimized'].width,variants['optimized'].height)==((1280,640) if size[0]>1280 else size)
for name in ('thumbnail.jpg','crop.jpg','optimized.webp'):
with Image.open(tmp_path/name) as image:
assert not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp')
if transparent:
if name.endswith('webp'):assert image.getpixel((0,0))[3]==0
else:assert min(image.getpixel((0,0)))>=250
async def test_exif_orientation_applied(tmp_path):
source=tmp_path/'source.jpg'
with Image.new('RGB',(400,200),'blue') as image:
exif=Image.Exif();exif[274]=6
image.save(source,exif=exif)
result=create_variants(source,tmp_path,'image/jpeg',40_000_000,ImageSettings(_env_file=None))
assert (result.variants[0].width,result.variants[0].height)==(128,256)
@pytest.mark.parametrize('failure', ['memory','cpu'])
async def test_worker_os_limits(failure,tmp_path):
body='return bytearray(2 * 1024 ** 3)' if failure=='memory' else 'while True: pass'
code="from app.core import thumbnail_worker as w\ndef fail(*args):\n "+body+"\nw.create_variants=fail\nraise SystemExit(w.main())"
result=await asyncio.to_thread(subprocess.run,[sys.executable,'-c',code,'source',str(tmp_path),str(tmp_path/'metadata'),'image/png','40000000'],
env={**os.environ,'IMAGE_MEMORY_MIB':'256','IMAGE_CPU_SECONDS':'1'},capture_output=True,timeout=15)
assert result.returncode==(3 if failure=='memory' else -9)
assert not (tmp_path/'metadata').exists()
async def test_render_timeout_kills_child(monkeypatch,tmp_path):
create=asyncio.create_subprocess_exec
children=[]
async def slow_child(*args,**kwargs):
child=await create(sys.executable,'-c','import time;time.sleep(60)',**kwargs)
children.append(child)
return child
monkeypatch.setattr(asyncio,'create_subprocess_exec',slow_child)
work=JobWork(uuid4(),uuid4(),uuid4(),1,'staging/test','image/png',1,'0'*64)
with pytest.raises(TimeoutError):
async with asyncio.timeout(.2):
await render_variants(MemoryStorage(),work,tmp_path/'source',tmp_path)
assert len(children)==1 and children[0].returncode is not None
+178
View File
@@ -0,0 +1,178 @@
"""Lab5: retry/replay, fencing, cleanup, limits и транзакции реального PostgreSQL."""
import asyncio
from datetime import timedelta
import errno
from pathlib import Path
from tempfile import TemporaryDirectory
from uuid import UUID, uuid4
import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.jobs import JobSettings
from app.domain import attachment_jobs
from app.domain.attachment_jobs import claim_job, process_attachment_job, cleanup_staging, build_result
from app.domain.job_admin import replay_failed_job
from app.domain.job_retry import classify_failure, retry_delay
from app.errors.exceptions import JobOperationError, ProcessingError
from app.models.attachment import Attachment
from app.models.job import AttachmentJob
from app.models.job_replay import JobReplay
from app.schemas.job import JobMessage
from app.storage import jobs
from test_attachments import context, upload, finish_attachment
async def envelope(session, attachment_id):
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
return JobMessage(job_id=job.id, attachment_id=job.attachment_id, pipeline_version=1,
attempt=job.attempt + 1, created_at=job.created_at)
async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
settings = JobSettings(_env_file=None, job_max_attempts=3)
def fail(_):
raise TimeoutError
async with AsyncSession(db_engine, expire_on_commit=False) as session:
first = await envelope(session, attachment['id'])
for attempt in range(1, 4):
message = await envelope(session, attachment['id'])
assert await process_attachment_job(session, storage, message, settings, before_processing=fail)
job, item = await jobs.lock_job(session, first.job_id)
assert job.attempt == attempt
delay = (job.next_attempt_at - item.updated_at).total_seconds()
assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1)
assert item.status == ('failed' if attempt == 3 else 'queued')
assert job.last_error_permanent is False
job.next_attempt_at = await jobs.database_time(session) - timedelta(seconds=1)
await session.commit()
assert item.error_code == 'retry_exhausted'
request_id = uuid4()
result = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id,
actor='test', reason='dependency_restored')
assert result['attempt_limit'] == 6
# Old envelope cannot consume the fresh replay budget.
assert await process_attachment_job(session, storage, first, settings)
job, item = await jobs.lock_job(session, first.job_id)
assert job.attempt == 3 and item.status == 'queued'
await session.commit()
message = await envelope(session, attachment['id'])
assert await process_attachment_job(session, storage, message, settings)
before = dict(storage.objects)
assert await process_attachment_job(session, storage, message, settings)
assert storage.objects == before
repeat = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id,
actor='test', reason='dependency_restored')
assert repeat['already_applied']
with pytest.raises(JobOperationError, match='job_not_failed'):
await replay_failed_job(session, storage, settings, first.job_id, request_id=uuid4(), actor='test', reason='manual_review')
await session.rollback()
assert len((await session.scalars(select(JobReplay))).all()) == 1
async def test_permanent_failure_and_missing_source_replay(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
def fail(_):
raise ProcessingError('image_limits')
settings = JobSettings(_env_file=None)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
await process_attachment_job(session, storage, message, settings, before_processing=fail)
job, item = await jobs.lock_job(session, message.job_id)
assert item.status == 'failed' and job.attempt == 1 and job.last_error_permanent
job.staging_key = None
await session.commit()
with pytest.raises(JobOperationError, match='source_expired'):
await replay_failed_job(session, storage, settings, job.id, request_id=uuid4(), actor='test', reason='manual_review')
async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
settings = JobSettings(_env_file=None)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
claim = await claim_job(session, message, settings)
job, item = await jobs.lock_job(session, message.job_id)
job.lease_token = uuid4()
job.finished_at = await jobs.database_time(session) - timedelta(days=10)
await session.commit()
assert await cleanup_staging(session, storage, settings, apply=True) == 0
# Rendering must not be allowed to PUT on a stale lease.
before = dict(storage.objects)
assert await build_result(session, storage, claim.work)
assert storage.objects == before
async def test_cleanup_ready_preserves_final_objects(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
await finish_attachment(client, storage, alice, attachment['id'])
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
job, item = await jobs.lock_job(session, message.job_id)
staging = job.staging_key
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
await session.commit()
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=False) == 1
assert staging in storage.objects
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 1
assert staging not in storage.objects and len(storage.objects) == 4
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 0
async def test_retry_classification_and_cap():
settings = JobSettings(_env_file=None)
for attempt in (1, 2, 6, 100):
cap = min(120, 5 * 2 ** min(30, attempt - 1))
assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100))
assert classify_failure(TimeoutError()) == ('processing_timeout', False)
assert classify_failure(OSError(errno.EACCES, 'denied'))[1]
assert not classify_failure(OSError(errno.ENOSPC, 'full'))[1]
async def test_timeout_records_retry_and_clears_lease(client, context, db_engine, monkeypatch):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
async def stalled(*args, **kwargs):
await asyncio.sleep(60)
monkeypatch.setattr(attachment_jobs, 'build_result', stalled)
settings = JobSettings(_env_file=None).model_copy(update={'job_timeout_seconds': .05})
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
assert await process_attachment_job(session, storage, message, settings)
job, item = await jobs.lock_job(session, message.job_id)
assert item.status == 'queued' and job.last_error_code == 'processing_timeout'
assert job.lease_token is None and job.lease_until is None
async def test_failed_retention_and_source_checksum_protect_replay(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
def fail(_):
raise ProcessingError('invalid_image')
settings = JobSettings(_env_file=None)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
await process_attachment_job(session, storage, message, settings, before_processing=fail)
job, item = await jobs.lock_job(session, message.job_id)
key = job.staging_key
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
await session.commit()
assert await cleanup_staging(session, storage, settings, apply=True) == 0
data, metadata = storage.objects[key]
storage.objects[key] = (b'x' * len(data), metadata)
with pytest.raises(JobOperationError, match='source_mismatch'):
await replay_failed_job(session, storage, settings, message.job_id, request_id=uuid4(), actor='test', reason='manual_review')
await session.rollback()
assert not (await session.scalars(select(JobReplay))).all()
job, item = await jobs.lock_job(session, message.job_id)
job.finished_at = await jobs.database_time(session) - timedelta(days=8)
await session.commit()
assert await cleanup_staging(session, storage, settings, apply=True) == 1
assert key not in storage.objects
+1 -1
View File
@@ -8,7 +8,7 @@ from sqlalchemy import inspect, text
from app.models import Base
from conftest import reset_schema, upgrade_head
EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents"}
EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents", "attachment_jobs", "job_replays"}
@pytest.fixture