diff --git a/.env.example b/.env.example index 4f28147..00fb270 100644 --- a/.env.example +++ b/.env.example @@ -11,7 +11,7 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id @@ -25,3 +25,4 @@ JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 MAX_PENDING_JOBS=100 +RENDER_MEMORY_MB=768 diff --git a/REPORT.md b/REPORT.md index 2aa0950..50b6b32 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,41 +1,173 @@ # Отчёт — лабораторная 5 -- Автор, группа: Matvey Kozyrev, BVT2301 -- Целевая оценка; выбранный трек 5, если нужен: 5, трек 1 «Очередь под перегрузкой» -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор: Матвей Козырев, BVT2301. +- Целевая оценка: 5, трек 1 «Очередь под перегрузкой». +- Ветка: `solution/BVT2301-Kozyrev`. Базовый commit: `4f6e69c` (`add level 5`); исправления критических ошибок и этот отчёт пока не закоммичены. Ссылка на PR не предоставлена. +- Дата локальной проверки: 26 октября 2026 года. +- Статус: код и локальные проверки подготовлены; выполнение всех требований уровней 3–5 пока не подтверждено. + +## Исходная база, источники и помощь ИИ + +Из собственной lab4 перенесены API на FastAPI, PostgreSQL-миграции, сессии, идемпотентность сообщений, S3-адаптер, две API-реплики и HAProxy. Основание проверки: [README](README.md), [семантика API](contracts/README.md), [состояния вложений](contracts/attachment-states.md), [OpenAPI](contracts/openapi.json), [инфраструктура](contracts/infrastructure.json), [публичные тесты](tests/README.md). + +Использован Codex для переноса исходников, подготовки реализации, аудита, исправлений и оформления отчёта. Локальные проверки не заменяют интеграционную демонстрацию. Справочные первоисточники задания: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), [publisher confirms и consumer acknowledgements](https://www.rabbitmq.com/docs/confirms). + +Зависимости зафиксированы в [pyproject.toml](pyproject.toml) и [uv.lock](uv.lock): FastAPI 0.141.1, Uvicorn 0.40.0, asyncpg 0.31.0, argon2-cffi 25.1.0, boto3 1.43.115, Pillow 13.0.0, pika 1.3.2. Имя пакета пока сохранено от lab4. ## Запуск из чистого clone -Python 3.12, PostgreSQL 17.6, SeaweedFS 4.17, RabbitMQ 4.1 и HAProxy 3.0 запускаются через Compose. Для локального синтетического стенда: `python3 scripts/init-secrets.py`, затем `docker compose up -d --build`; дождаться `docker compose ps` со статусом healthy и выполнить `make test`. Локальные секреты создаются в игнорируемом `secrets/`. Остановка без удаления данных: `docker compose down`; удаление учебных данных выполняется оператором отдельно через `docker compose down -v`. +Нужны Git, Python 3.12+ для приложения, uv и работающий Docker Engine с Compose v2. REST доступен на `127.0.0.1:8080`; порт должен быть свободен, в том числе от стенда lab4. Используются синтетические данные. + +В [Dockerfile](Dockerfile) и [Compose](compose.yaml) закреплены Python 3.12.12-slim, uv 0.11.28, PostgreSQL 17.6-alpine, SeaweedFS 4.17 с digest, RabbitMQ 4.1.2-alpine, HAProxy 3.0.11-alpine. + +Из корня полученного clone: + +```sh +git switch solution/BVT2301-Kozyrev +python3 scripts/init-secrets.py +docker compose config --quiet +docker compose up -d --build +docker compose ps -a +curl --fail http://localhost:8080/health/ready +make test +``` + +Локальные credentials создаются в игнорируемом `secrets/`. Их содержимое нельзя публиковать. Стандартный Compose использует значения по умолчанию; отдельный .env не обязателен. Сервис migrate применяет SQL из migrations, s3-init создаёт приватный bucket и проверяет запись. API и worker зависят от успешного завершения этих сервисов. + +Эта инструкция пока не подтверждена запуском из чистого clone. Успешный `up -d` сам по себе не доказывает готовность: нужны проверка процессов, readiness и smoke. Worker использует heartbeat AMQP event loop; outbox/reconcile не наследуют HTTP healthcheck. + +Локальные проверки: + +```sh +uv sync --locked --no-dev +make check +.venv/bin/python student-tests/pipeline_critical.py +# Для полной OpenAPI-валидации: +uv pip install --python .venv/bin/python -r requirements-dev.txt +make validate PYTHON=.venv/bin/python +``` + +Остановка с сохранением volumes: `docker compose down`. Намеренный сброс учебных данных: `docker compose down -v`; удаляет volumes PostgreSQL, S3 и RabbitMQ данного проекта. В ходе проверки сброс не выполнялся. ## Архитектура и решения -PostgreSQL хранит метаданные, jobs и transactional outbox; SeaweedFS — private staging/final objects; RabbitMQ — durable сообщения; worker и outbox-dispatcher — отдельные процессы. До HTTP 202 API полностью принимает и валидирует bytes, сохраняет staging-object и в одной DB-транзакции метаданные + job + outbox event. Dispatcher публикует событие с publisher confirm и отмечает event после confirm, поэтому crash между COMMIT и publish восстанавливается без ручной job. Worker ack делает только после durable результата либо после финального failed. Обработка идемпотентна по attachment/pipeline/variant: повтор ready-job не меняет результат. Lease ограничен `JOB_TIMEOUT_SECONDS`; истёкшая processing-job возвращается в очередь dispatcher-ом. Retry ограничен `JOB_MAX_ATTEMPTS`, использует exponential backoff с jitter; исчерпанная job остаётся failed и её служебное сообщение попадает в DLQ. Явный replay: `docker compose run --rm outbox /app/.venv/bin/python -m src.replay `. +Клиент → HAProxy → API → PostgreSQL + S3 staging. Фоновый путь: PostgreSQL outbox → dispatcher → RabbitMQ → worker → final S3 objects + метаданные PostgreSQL. Реализация: [attachments.py](src/attachments.py), [postgres.py](src/storage/postgres.py), [outbox.py](src/outbox.py), [broker.py](src/broker.py), [worker.py](src/worker.py). -На уровне 4 worker создаёт: thumbnail JPEG 256×256 без увеличения; crop JPEG 128×128, центрированный квадрат; optimized WebP до 1280×1280 без увеличения. JPEG quality=85, WebP quality=80; входные EXIF/metadata не сохраняются. +PostgreSQL хранит пользователей, чаты, сообщения, сессии, attachments, jobs и outbox. S3 хранит байты. Брокер передаёт служебные ID, без файлов и credentials. -## Доказательства критериев +Перед 202 API полностью принимает HTTP body, проверяет изображение, записывает исходник в staging и фиксирует attachment/job/outbox одной DB-транзакцией. S3 PUT не входит в DB-транзакцию: при rollback возможен непривязанный объект. Resize выполняет worker. Клиент получает состояние через GET метаданных; content неготового вложения возвращает 409. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Трек 5: очередь под перегрузкой | `make track1 TRACK_ARGS="..."` | TODO: заполнить только результатом выполненного прогона | `evidence/track1-*.json` | +Dispatcher декодирует JSONB и публикует persistent сообщение с publisher confirm, затем отмечает event опубликованным. Crash после confirm до отметки допускает дубль. Автоматическое восстановление после реального crash ещё не проверено. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Worker захватывает job с lease. Завершение проверяет состояние, время захвата processing_started_at и срок lease. Попытки пишут по разным S3-ключам; только действующая попытка публикует свои ключи в БД. Устаревшее завершение не должно менять актуальный результат. Логический ключ варианта — (attachment_id, pipeline_version, name); физические непривязанные объекты требуют уборки. -## Эксперименты и отказы +Ack выполняется после устойчивого результата или записи retry/failed; при сбое записи используется nack/requeue. Terminal-job повторно не обрабатывается. Это at-least-once доставка, не exactly-once. Реестр failed хранится в PostgreSQL; DLQ используется для некорректных сообщений. -Гипотеза: увеличение worker concurrency сокращает p95 «202 → ready» до точки насыщения CPU/RAM; prefetch выше concurrency может повысить throughput ценой queue lag и RSS. Проверяются минимум три конфигурации: `1/1`, `2/2`, `4/4` (concurrency/prefetch) и несколько размеров входного PNG. Для каждой конфигурации используются одинаковые CPU/RAM limits Compose, одинаковое число изображений и не менее трёх прогонов после прогрева. +Ошибка удаления staging после ready журналируется и не откатывает результат. Сервис reconcile раз в минуту удаляет непривязанные объекты старше retention из staging/originals/variants. Общая advisory-блокировка исключает удаление во время upload и работы worker. Staging failed-задач сохраняется для явного replay; автоматического удаления этих исходников нет. Lease recovery учитывает попытки; retry использует exponential backoff с jitter. Невалидное изображение и превышение памяти рендера переводят задачу в failed без повторов. Ошибки зависимостей и timeout повторяются с ограничением попыток. -Команда запуска профиля: `WORKER_CONCURRENCY=N WORKER_PREFETCH=P MAX_PENDING_JOBS=20 docker compose up -d --build`. Команда измерения: `make track1 TRACK_ARGS="--run-name cN-pP-sizeS --workers N --prefetch P --max-pending 20 --images 40 --concurrency 8 --size S"`. Скрипт сохраняет raw JSON с successful throughput, p50/p95 времени принятия и ready, queue depth RabbitMQ и RSS worker. Значения будут внесены после реального прогона; сейчас их нет, так как Docker daemon недоступен. +Явный операторский replay только failed-job: -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +```sh +docker compose run --rm outbox /app/.venv/bin/python -m src.replay ATTACHMENT_UUID +``` + +Replay сохраняет replayed_at; полного журнала операторских действий нет. + +## Конфигурация и варианты изображений + +| Параметр | Соответствие реализации | +| --- | --- | +| SESSION_TTL_SECONDS / IDEMPOTENCY_TTL_SECONDS | По умолчанию 3600 / 86400 секунд, PostgreSQL | +| S3_ENDPOINT | Compose и .env.example используют http://s3:8333 | +| DATABASE_URL_FILE, CURSOR_SECRET_FILE, S3/Broker *_FILE | Credentials подключаются через Compose secrets | +| MAX_UPLOAD_BYTES / MAX_IMAGE_PIXELS | 10 485 760 байт / 40 000 000 пикселей; 10 MiB = 10 × 1024 × 1024 байт | +| WORKER_CONCURRENCY / WORKER_PREFETCH | По умолчанию 2 / 2, потоки и prefetch | +| JOB_MAX_ATTEMPTS / JOB_TIMEOUT_SECONDS | По умолчанию 5 / 60; рендер идёт в отдельном subprocess, принудительно завершаемом по timeout | +| MAX_PENDING_JOBS | По умолчанию 100 queued/processing jobs; глобальная DB-блокировка при admission, отказ 429 | +| STAGING_RETENTION_SECONDS | Порог возраста непривязанных объектов для плановой уборки, по умолчанию 86400 секунд | +| API_REPLICAS, SESSION_STORE, CACHE_URL_FILE | Не управляют текущим Compose: явно заданы api-a/api-b и PostgreSQL без Redis | + +Worker применяет EXIF orientation, композитит alpha на белый, создаёт thumbnail JPEG до 256×256, центрированный crop JPEG 128×128, optimized WebP до 1280×1280. JPEG quality=85, WebP quality=80/method=4; явного переноса входных EXIF нет. + +Округление исправлено: исходник 300×200 даёт 256×170; высота вычисляется как floor(200 × 256 / 300). Маленькие изображения не увеличиваются. Linux renderer ограничен RLIMIT_AS=768 MiB; worker-контейнер ограничен 4 GiB RAM и 2 CPU. RSS собирается отдельно, RLIMIT_AS ограничивает виртуальную память. + +## Выполненные проверки и доказательства критериев + +Повторная локальная проверка 26 октября 2026 года: + +```text +make check +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +.venv/bin/python student-tests/pipeline_critical.py +Ran 8 tests +OK + +docker compose config --quiet +Завершено с кодом 0. + +docker info --format '{{.ServerVersion}}' +Ошибка: Docker socket отсутствует; daemon недоступен. +``` + +[Регрессионные тесты](student-tests/pipeline_critical.py) используют mocks и проверяют декодирование outbox, ack/nack, confirm DLQ, сохранение ready при ошибке cleanup и отклонение устаревшего завершения/retry. Они не проверяют SQL на реальной БД. Полная OpenAPI-валидация при аудите не запустилась из-за отсутствующего openapi_spec_validator. + +| Критерий README | Состояние / необходимая проверка | +| --- | --- | +| 3: минимум lab4 и durable upload 202 | Код перенесён; контейнерный smoke не выполнен; failed-вложение отклоняется с 409 | +| 3: отдельные broker/worker, durable доставка, ack | Роли/volume/persistent publish есть; локальные ack/nack-тесты прошли; реальная redelivery не проверена | +| 3: состояния, оригинал, thumbnail, smoke | Реализованы; округление проверено локально; успешного smoke нет | +| 3: ограниченные retries, failed, идемпотентность | Счётчик, terminal states и защита завершения есть; конкурентные проверки на БД не выполнены | +| 3: worker stop → upload → restart API/broker → ready | Не выполнено; сохранность при рестарте не доказана | +| 4: crop/WebP и лимиты bytes/pixels/time/memory | Варианты и bytes/pixels есть; subprocess timeout и лимиты памяти добавлены; Linux-лимиты на стенде не проверены | +| 4: outbox, crash COMMIT/publish | Код и OUTBOX_FAULT_AFTER_COMMIT=1 есть; crash recovery не проверен | +| 4: backoff/jitter, классификация, failed/replay | Backoff/replay есть; классификация добавлена; replay на стенде не выполнен | +| 4: crash после side effect, lease, concurrency, cleanup | Защита завершения и concurrency есть; crash-тест отсутствует; плановый cleanup реализован; failed staging сохраняется для replay | +| 5: три настройки, разные изображения, метрики | Генератор исправлен и проверен с mock-стендом; реальных прогонов нет | +| 5: backpressure и восстановление | Лимит jobs есть; демонстрация перегрузки и дренирования не выполнена | + +В evidence пока нет результатов контейнерного smoke или исследования. Наличие кода не считается доказательством выполнения критерия. + +## Эксперимент: очередь под перегрузкой + +Гипотеза: рост concurrency увеличивает throughput до насыщения ресурсов; затем задержки и память могут вырасти. Экспериментальных выводов пока нет. + +План эксперимента: профили concurrency/prefetch 1/1, 2/2, 4/4; изображения 320×320, 1600×1600, 3200×3200; одинаковые CPU/RAM-лимиты; отдельный прогрев и три повторения каждой комбинации. Фиксировать hardware и реальные настройки контейнеров, длительность, accepted/rejected/failed, throughput, queue waiting time, p50/p95/p99 времени 202 → ready и RSS. CPU отдельно фиксировать по контейнерным метрикам. + +Заготовка запуска: + +```sh +WORKER_CONCURRENCY=1 WORKER_PREFETCH=1 MAX_PENDING_JOBS=20 docker compose up -d --build +make track1 TRACK_ARGS="--run-name c1-p1-s1600 --workers 1 --prefetch 1 --max-pending 20 --images 40 --concurrency 8 --size 1600" +``` + +Ожидаемый путь результата этой команды: evidence/c1-p1-s1600.json. Измерения ещё не получены. + +Исправленный [генератор](scripts/track1.py) записывает monotonic-время ответа каждой загрузки и опрашивает ready параллельно uploads. Барьер удалён. RSS суммируется по /proc процессов worker-контейнера; shared pages могут учитываться несколько раз, пики между замерами пропускаются. Время ожидания вычисляется в PostgreSQL от создания job до первого claim (миграция 007). Concurrency/prefetch и лимит admission сверяются с работающими контейнерами. Closed-loop и polling вносят ограничения: наблюдаемое время ready является верхней оценкой, а нагрузка зависит от скорости клиента. + +После нагрузки требуется проверить завершение всех принятых задач, отсутствие queued/processing и unacked, затем обработать контрольный upload. Сценарий проверяет дренирование и контрольный upload; при незавершённых задачах или пропущенных метриках завершается ошибкой. Флаг --require-backpressure дополнительно требует наблюдаемый 429. Измеренных RPS, p95, RSS и времени восстановления нет; ресурсы Docker не зафиксированы из-за недоступного daemon. + +## Эксперименты с отказами + +| Воздействие | Ожидаемый эффект | Наблюдение | +| --- | --- | --- | +| Worker остановлен | 202, metadata, 409 content; ready после запуска | Не выполнено | +| API погиб после COMMIT | Dispatcher доставляет intent | Не выполнено | +| Worker погиб после S3 до commit/ack | Lease recovery завершает ту же логическую работу | Не выполнено | +| Сбой cleanup после ready | Ready сохраняется | Локальный mock-тест прошёл | +| Сбой устойчивой записи результата/retry | nack/requeue | Локальный mock-тест прошёл | +| Устаревшее завершение | Метаданные не меняются, staging сохраняется | Локальные mock-тесты прошли | +| Broker restart с volume | Принятая работа сохраняется | Не выполнено | +| Перегрузка | Ограничение приёма, дренирование, восстановление | Не выполнено | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Перечисленные в аудите нарушения исправлены в коде. Требуются проверки на реальных PostgreSQL/S3/RabbitMQ: Linux memory limit, конкурентная уборка, рестарты и нагрузочные измерения пока не подтверждены. + +Admission перенесён перед S3 PUT; отклонённые по лимиту uploads не создают S3-объект. Reconciler адаптирован к вариантам и блокировке активных writers. Тест student-tests/attachments.py обновлён для 202 и polling. Другие исторические тесты lab4 требуют отдельной адаптации и не заявляются как пройденные. + +Один Docker-host, одна БД, один брокер, одно S3-хранилище и один LB не доказывают устойчивость к потере host/хранилищ. Две API-реплики покрывают только часть отказов. Порядок завершения: закрыть замечания аудита, запустить smoke и persistence/failover, воспроизвести отказы pipeline, исправить генератор и провести исследование. До этого оценка 5 остаётся целью, а не подтверждённым результатом. + +### Дополнительные локальные регрессии + +`.venv/bin/python student-tests/remaining_regressions.py`: проверены округление/alpha, subprocess render и его timeout, классификация permanent failure, retention активного/failed staging, частичная группа нагрузочного генератора и контроль восстановления на mock-стенде. Числа mock-прогона не являются результатами исследования и не переносятся в evidence. diff --git a/compose.yaml b/compose.yaml index b040b76..2bc3704 100644 --- a/compose.yaml +++ b/compose.yaml @@ -151,10 +151,13 @@ services: memory: 128M reconcile: + healthcheck: + disable: true build: context: . entrypoint: ["/app/.venv/bin/python", "-m", "src.reconcile"] - profiles: ["maintenance"] + command: ["--apply", "--loop"] + restart: unless-stopped environment: DATABASE_URL_FILE: /run/secrets/database_url CURSOR_SECRET_FILE: /run/secrets/cursor_secret @@ -162,6 +165,7 @@ services: S3_BUCKET: messenger-attachments S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} secrets: - database_url - cursor_secret @@ -185,6 +189,17 @@ services: <<: *api worker: + restart: unless-stopped + healthcheck: + test: ["CMD", "/app/.venv/bin/python", "-c", "import time,pathlib; assert time.time()-float(pathlib.Path('/tmp/worker-heartbeat').read_text()) < 15"] + interval: 5s + timeout: 3s + retries: 3 + deploy: + resources: + limits: + cpus: "2.0" + memory: 4G build: context: . command: ["/app/.venv/bin/python", "-m", "src.worker"] @@ -198,6 +213,7 @@ services: S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key BROKER_URL_FILE: /run/secrets/broker_url WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + RENDER_MEMORY_MB: ${RENDER_MEMORY_MB:-768} WORKER_PREFETCH: ${WORKER_PREFETCH:-2} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} @@ -218,6 +234,8 @@ services: stop_grace_period: 30s outbox: + healthcheck: + disable: true build: context: . command: ["/app/.venv/bin/python", "-m", "src.outbox"] @@ -227,6 +245,7 @@ services: CURSOR_SECRET_FILE: /run/secrets/cursor_secret BROKER_URL_FILE: /run/secrets/broker_url OUTBOX_DISPATCH_INTERVAL_SECONDS: "1" + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} secrets: - database_url - cursor_secret diff --git a/migrations/007_job_measurements.sql b/migrations/007_job_measurements.sql new file mode 100644 index 0000000..12fc193 --- /dev/null +++ b/migrations/007_job_measurements.sql @@ -0,0 +1 @@ +ALTER TABLE attachment_jobs ADD COLUMN first_started_at timestamptz; diff --git a/scripts/restart.sh b/scripts/restart.sh index 647ec46..958b700 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -6,5 +6,5 @@ cd "$(dirname "$0")/.." docker compose restart postgres s3 broker docker compose up -d --wait postgres s3 broker docker compose run --rm --no-deps s3-init -docker compose restart worker api-a api-b load_balancer -docker compose up -d --wait worker api-a api-b load_balancer +docker compose restart worker outbox reconcile api-a api-b load_balancer +docker compose up -d --wait worker outbox reconcile api-a api-b load_balancer diff --git a/scripts/track1.py b/scripts/track1.py index eb3d238..f8ff5b2 100755 --- a/scripts/track1.py +++ b/scripts/track1.py @@ -1,13 +1,5 @@ #!/usr/bin/env python3 -"""Repeatable level-5 queue-overload experiment. - -Run a Compose profile first, for example: - WORKER_CONCURRENCY=2 MAX_PENDING_JOBS=20 docker compose up -d --build - .venv/bin/python scripts/track1.py --run-name c2-p2 --workers 2 --prefetch 2 --images 40 --size 1600 - -The script records client-observed acceptance/ready latency, RabbitMQ queue depth -and Docker RSS samples. It does not print credentials or session tokens. -""" +"""Closed-loop upload burst with concurrent readiness observations and drain proof.""" from concurrent.futures import ThreadPoolExecutor, as_completed from io import BytesIO import json @@ -17,11 +9,10 @@ import subprocess import sys import threading import time -from urllib.parse import quote ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) -from client import Client, Failure, arguments, check, run # noqa: E402 +from client import Client, Failure, arguments, check, run def percentile(values, point): @@ -29,114 +20,194 @@ def percentile(values, point): return round(values[max(0, math.ceil(len(values) * point) - 1)] * 1000, 3) if values else None -def docker_text(*args): +def command_json(*args): + result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=20) + if result.returncode: + raise Failure("metrics command failed: " + args[0]) try: - return subprocess.run(args, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, - timeout=10, check=True).stdout.strip() - except (OSError, subprocess.SubprocessError): - return None + return json.loads(result.stdout) + except ValueError: + raise Failure("metrics command returned invalid JSON") from None + + +def metrics(chat_id): + return command_json("docker", "compose", "exec", "-T", "worker", + "/app/.venv/bin/python", "-m", "src.pipeline_metrics", chat_id) def broker_depth(): - text = docker_text("docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "list_queues", "name", "messages_ready", "messages_unacknowledged", "--formatter", "json") - if not text: - return None - try: - rows = json.loads(text) - return next((row for row in rows if row["name"] == "attachment.process.v1"), None) - except (ValueError, KeyError, TypeError): - return None + rows = command_json("docker", "compose", "exec", "-T", "broker", + "rabbitmqctl", "list_queues", "name", "messages_ready", + "messages_unacknowledged", "--formatter", "json") + return next(row for row in rows if row["name"] == "attachment.process.v1") -def worker_rss(): - container = docker_text("docker", "compose", "ps", "-q", "worker") - if not container: - return None - text = docker_text("docker", "stats", "--no-stream", "--format", "{{json .}}", container) - if not text: - return None - try: - return json.loads(text).get("MemUsage") - except ValueError: - return None +def container_info(service): + result = subprocess.run(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, + capture_output=True, text=True, timeout=10, check=True) + ids = result.stdout.split() + check(len(ids) == 1, 'expected exactly one container: ' + service) + return command_json('docker', 'inspect', ids[0])[0] def main(): - def configure(parser): - parser.add_argument("--run-name", required=True) - parser.add_argument("--workers", type=int, required=True, help="documented WORKER_CONCURRENCY setting") - parser.add_argument("--prefetch", type=int, required=True, help="documented RabbitMQ prefetch setting") - parser.add_argument("--max-pending", type=int, required=True, help="documented MAX_PENDING_JOBS setting") - parser.add_argument("--images", type=int, default=40) - parser.add_argument("--concurrency", type=int, default=4) - parser.add_argument("--size", type=int, default=1600, help="square synthetic PNG dimensions") - parser.add_argument("--max-ready-seconds", type=float, default=120) - parser.add_argument("--output", default=None) - args = arguments("Lab 5 track 1 image-pipeline experiment", configure) - check(1 <= args.workers <= 64 and 1 <= args.prefetch <= 1024 and 1 <= args.max_pending <= 100000, "invalid worker/prefetch/max-pending") - check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128 and 1 <= args.size <= 6000, "invalid workload") - + def configure(p): + p.add_argument("--run-name", required=True) + p.add_argument("--workers", type=int, required=True) + p.add_argument("--prefetch", type=int, required=True) + p.add_argument("--max-pending", type=int, required=True) + p.add_argument("--images", type=int, default=40) + p.add_argument("--concurrency", type=int, default=8) + p.add_argument("--size", type=int, default=1600) + p.add_argument("--max-ready-seconds", type=float, default=180) + p.add_argument("--require-backpressure", action="store_true") + p.add_argument("--output") + args = arguments(__doc__, configure) + check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128, "invalid workload") + check(1 <= args.size <= 6000 and 0 < args.max_ready_seconds <= 3600, "invalid size/deadline") + check(args.workers > 0 and args.prefetch > 0 and args.max_pending > 0, "invalid profile") from PIL import Image - image = Image.new("RGB", (args.size, args.size), "#427aa1") - raw = BytesIO(); image.save(raw, "PNG", optimize=True); data = raw.getvalue() + # Reproducible, nonuniform synthetic image; no real personal data. + row = bytes(v for x in range(args.size) for v in ((x * 13) % 256, (x * 37) % 256, (x * 71) % 256)) + image = Image.frombytes("RGB", (args.size, args.size), row * args.size) + stream = BytesIO() + image.save(stream, "PNG") + data = stream.getvalue() client = Client(args.base_url, args.ca_file, timeout=30) - alice, bob, _eve, chat = client.bootstrap() + alice, bob, _, chat = client.bootstrap() + baseline = metrics(chat["id"]) + check(baseline["concurrency"] == args.workers and baseline["prefetch"] == args.prefetch, + "worker settings do not match requested profile") + check(baseline["pending"] == 0, "use an idle isolated stack") + for service in ("api-a", "api-b"): + env = dict(item.split('=', 1) for item in container_info(service)['Config']['Env']) + check(int(env['MAX_PENDING_JOBS']) == args.max_pending, + 'running API MAX_PENDING_JOBS does not match profile') + resources = container_info('worker')['HostConfig'] path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=track1.png" - barrier = threading.Barrier(args.concurrency) - queue_samples, rss_samples = [], [] - stop_sampling = threading.Event() - - def sampler(): - while not stop_sampling.wait(1): - queue_samples.append({"seconds": round(time.monotonic() - started, 3), "queue": broker_depth()}) - rss_samples.append({"seconds": round(time.monotonic() - started, 3), "rss": worker_rss()}) - + pending, records, samples, errors = {}, [], [], [] + lock, finished = threading.Lock(), threading.Event() + started = time.monotonic() + def observe(): + local = Client(args.base_url, args.ca_file, timeout=5) + while not finished.wait(.1): + with lock: + batch = list(pending.items()) + for key, record in batch: + if finished.is_set(): + break + try: + att = local.call("GET", "/api/v1/attachments/" + key, actor=bob, schema="Attachment")["json"] + if att["status"] in ("ready", "failed"): + with lock: + record["terminal"] = att["status"] + record["terminal_at"] = time.monotonic() + pending.pop(key, None) + except Failure: + continue + def sample(): + while not finished.is_set(): + try: + sample = metrics(chat["id"]) + sample["queue"] = broker_depth() + sample["seconds"] = time.monotonic() - started + with lock: + samples.append(sample) + except (Failure, OSError, subprocess.TimeoutExpired) as exc: + with lock: + errors.append(type(exc).__name__) + if finished.wait(1): + break + observer = threading.Thread(target=observe, daemon=True) + sampler = threading.Thread(target=sample, daemon=True) + observer.start() + sampler.start() def upload(index): local = Client(args.base_url, args.ca_file, timeout=30) - barrier.wait() - started_at = time.monotonic() + before = time.monotonic() try: - response = local.call("POST", path + "-" + str(index), 202, actor=alice, raw=data, + response = local.call("POST", path, (202, 429), actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment") - return {"accepted": True, "id": response["json"]["id"], "accepted_seconds": time.monotonic() - started_at} - except Failure as error: - return {"accepted": False, "error": str(error), "accepted_seconds": time.monotonic() - started_at} - - started = time.monotonic() - thread = threading.Thread(target=sampler, daemon=True); thread.start() - with ThreadPoolExecutor(max_workers=args.concurrency) as pool: - results = list(pool.map(upload, range(args.images))) - accepted = [item for item in results if item["accepted"]] - ready_latencies, failed_ready = [], 0 - deadline = time.monotonic() + args.max_ready_seconds - pending = {item["id"]: started + item["accepted_seconds"] for item in accepted} - while pending and time.monotonic() < deadline: - for attachment_id, accepted_at in list(pending.items()): - try: - metadata = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=bob, schema="Attachment")["json"] - except Failure: - continue - if metadata["status"] == "ready": - ready_latencies.append(time.monotonic() - accepted_at) - del pending[attachment_id] - elif metadata["status"] == "failed": - failed_ready += 1; del pending[attachment_id] - time.sleep(.2) - stop_sampling.set(); thread.join(timeout=2) - elapsed = time.monotonic() - started + received = time.monotonic() + record = {"index": index, "status": response["status"], "upload_seconds": received - before} + if response["status"] == 202: + record.update(id=response["json"]["id"], accepted_at=received) + with lock: + records.append(record) + if response["status"] == 202: + pending[record["id"]] = record + except Failure: + with lock: + records.append({"index": index, "status": "transport_or_http_error"}) + try: + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + list(pool.map(upload, range(args.images))) + upload_finished = time.monotonic() + deadline = upload_finished + args.max_ready_seconds + while time.monotonic() < deadline: + with lock: + empty = not pending + if empty: + break + time.sleep(.1) + finally: + measured_end = time.monotonic() + finished.set() + observer.join(timeout=10) + sampler.join(timeout=45) + elapsed = measured_end - started + final = metrics(chat["id"]) + queue = broker_depth() + for _ in range(20): + if queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0: + break + time.sleep(.25) + queue = broker_depth() + accepted = [r for r in records if r["status"] == 202] + ready = [r for r in accepted if r.get("terminal") == "ready"] + latencies = [r["terminal_at"] - r["accepted_at"] for r in ready] + waits = [r["queue_wait_seconds"] for r in final["jobs"] if r["queue_wait_seconds"] is not None] + drained = (bool(accepted) and len(ready) == len(accepted) and final["pending"] == 0 + and queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0) + recovery = False + if drained: + try: + att, _ = client.attachment(alice, chat) + client.wait_ready(bob, att, wait_seconds=args.max_ready_seconds) + recovery = True + except Failure: + pass + rejected = sum(r["status"] == 429 for r in records) report = { - "track": "queue-overload", "run_name": args.run_name, - "settings": {"worker_concurrency": args.workers, "prefetch": args.prefetch, "max_pending_jobs": args.max_pending}, - "workload": {"images": args.images, "client_concurrency": args.concurrency, "png_dimensions": [args.size, args.size], "input_bytes": len(data)}, - "results": {"elapsed_seconds": round(elapsed, 3), "accepted": len(accepted), "rejected_or_failed": args.images - len(accepted), "ready": len(ready_latencies), "failed": failed_ready, "timed_out": len(pending), "throughput_ready_per_second": round(len(ready_latencies) / elapsed, 4), "accept_latency_ms": {"p50": percentile([x["accepted_seconds"] for x in accepted], .5), "p95": percentile([x["accepted_seconds"] for x in accepted], .95)}, "time_to_ready_ms": {"p50": percentile(ready_latencies, .5), "p95": percentile(ready_latencies, .95)}}, - "queue_samples": queue_samples, "worker_rss_samples": rss_samples, - "limitations": "Closed-loop client; queue/RSS samples are null when Docker CLI is unavailable. p95 covers ready tasks only.", + "run_name": args.run_name, + "settings": {"workers": baseline["concurrency"], "prefetch": baseline["prefetch"], + "max_pending": args.max_pending, + "worker_limits": {key: resources[key] for key in ('Memory', 'NanoCpus')}}, + "workload": {"images": args.images, "concurrency": args.concurrency, + "size": args.size, "bytes": len(data), "type": "closed-loop burst"}, + "results": {"accepted": len(accepted), "rejected_429": rejected, "ready": len(ready), + "failed": sum(r.get("terminal") == "failed" for r in accepted), + "timed_out": len(pending), "elapsed_seconds": elapsed, + "ready_per_second": len(ready) / elapsed, + "ready_latency_ms": {str(p): percentile(latencies, p) for p in (.5, .95, .99)}, + "queue_wait_ms": {str(p): percentile(waits, p) for p in (.5, .95, .99)}, + "max_sampled_rss_bytes": max((s["rss_bytes"] for s in samples), default=None), + "drained": drained, "recovery_upload_ready": recovery}, + "records": [{k: v for k, v in r.items() if k not in ("accepted_at", "terminal_at")} | + ({"ready_seconds": r["terminal_at"] - r["accepted_at"]} if "terminal_at" in r else {}) + for r in records], + "samples": samples, "metric_errors": errors, + "limitations": "Closed-loop; ready time is polling upper bound. RSS is sum of worker/container processes, shared pages may be counted twice. Sampling may miss peaks. Queue wait is first claim minus durable job creation.", } - text = json.dumps(report, ensure_ascii=False, indent=2) - destination = Path(args.output or (ROOT / "evidence" / (args.run_name + ".json"))) - destination.parent.mkdir(parents=True, exist_ok=True); destination.write_text(text + "\n", encoding="utf-8") - print(text) - check(bool(accepted), "no accepted uploads") + target = Path(args.output or ROOT / "evidence" / (args.run_name + ".json")) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report["results"], indent=2)) + check(not errors and bool(samples), "incomplete metrics; result saved but not valid evidence") + check(all(r["status"] in (202, 429) for r in records), "unexpected HTTP/transport failures") + check(drained and recovery, "pipeline did not drain/recover; see saved result") + if args.require_backpressure: + check(rejected > 0, "backpressure was not observed") if __name__ == "__main__": diff --git a/src/attachments.py b/src/attachments.py index 17c0cb8..ea496ee 100644 --- a/src/attachments.py +++ b/src/attachments.py @@ -83,10 +83,6 @@ async def upload(chat_id: UUID, request: Request, filename: str = Query(...), attachment_id, job_id = uuid4(), uuid4() key = f"staging/{attachment_id}/original" async with store.attachment_transaction() as connection: - await s3.put(key, file, size, content_type) - # Disabled by default; models a crash after PUT, before INSERT. - if os.environ.get("UPLOAD_FAULT_AFTER_S3_PUT") == "1": - raise ApiError(503, "unavailable", "Инъецирован отказ после S3 PUT") try: max_pending = int(os.environ.get("MAX_PENDING_JOBS", "100")) except ValueError as error: @@ -94,6 +90,9 @@ async def upload(chat_id: UUID, request: Request, filename: str = Query(...), if max_pending < 1: raise ApiError(503, "unavailable", "Некорректный лимит очереди") await store.ensure_attachment_queue_capacity(connection, max_pending) + await s3.put(key, file, size, content_type) + if os.environ.get("UPLOAD_FAULT_AFTER_S3_PUT") == "1": + raise ApiError(503, "unavailable", "Инъецирован отказ после S3 PUT") attachment = await store.create_attachment( attachment_id, chat_id, user["id"], filename, content_type, size, digest.hexdigest(), key, job_id, connection=connection, @@ -130,9 +129,12 @@ async def download(attachment_id: UUID, request: Request, if not matching: raise ApiError(404, "not_found", "Вариант не найден") item = matching[0] - suffix = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"}[variant] + object_key = await store.pool.fetchval( + 'SELECT object_key FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=$2 AND name=$3', + attachment_id, attachment['pipeline_version'], variant, + ) data = await request.app.state.s3.download_key( - f"variants/{attachment_id}/{suffix}", item["size_bytes"], item["sha256"] + object_key, item["size_bytes"], item["sha256"] ) content_type = item["content_type"] return Response(data, media_type=content_type, diff --git a/src/broker.py b/src/broker.py index 3e5b126..4cd980d 100644 --- a/src/broker.py +++ b/src/broker.py @@ -51,7 +51,7 @@ async def publish(payload: dict) -> None: async def publish_dlq(payload: dict) -> None: - payload = {**payload, "failure": "retry_exhausted"} + payload = {**payload, "failure": payload.get('failure', 'retry_exhausted')} try: await run_in_threadpool(_publish_to, DLQ, payload) except (pika.exceptions.AMQPError, OSError, ValueError) as error: @@ -61,9 +61,12 @@ async def publish_dlq(payload: dict) -> None: def _publish_to(queue: str, payload: dict) -> None: connection = _connection() try: - connection.channel().basic_publish( + channel = connection.channel() + channel.confirm_delivery() + channel.basic_publish( exchange="", routing_key=queue, body=json.dumps(payload, separators=(",", ":")).encode("utf-8"), properties=pika.BasicProperties(delivery_mode=pika.DeliveryMode.Persistent, content_type="application/json"), + mandatory=True, ) finally: connection.close() diff --git a/src/config.py b/src/config.py index 447b9ba..320063a 100644 --- a/src/config.py +++ b/src/config.py @@ -22,6 +22,7 @@ _CONFIGURATION_KEYS = { "MAX_UPLOAD_BYTES", "MAX_IMAGE_PIXELS", "UPLOAD_CONCURRENCY", "UPLOAD_TIMEOUT_SECONDS", "UPLOAD_FAULT_AFTER_S3_PUT", "WORKER_CONCURRENCY", "JOB_MAX_ATTEMPTS", "JOB_TIMEOUT_SECONDS", "STAGING_RETENTION_SECONDS", + "WORKER_PREFETCH", "MAX_PENDING_JOBS", "RENDER_MEMORY_MB", } diff --git a/src/outbox.py b/src/outbox.py index 340928e..60f0099 100644 --- a/src/outbox.py +++ b/src/outbox.py @@ -18,7 +18,7 @@ async def run() -> None: try: while True: try: - await store.schedule_due_jobs() + await store.schedule_due_jobs(int(os.environ.get('JOB_MAX_ATTEMPTS', '5'))) for event in await store.due_outbox_events(): await publish(event["payload"]) await store.mark_outbox_published(event["id"]) diff --git a/src/pipeline_metrics.py b/src/pipeline_metrics.py new file mode 100644 index 0000000..93cde0b --- /dev/null +++ b/src/pipeline_metrics.py @@ -0,0 +1,37 @@ +"""Operator-only metrics for one synthetic chat; never exposes credentials.""" +import asyncio +import json +import os +from pathlib import Path +import sys +from uuid import UUID +from .config import load_settings +from .storage.postgres import PostgresStore + + +async def main(chat): + store = PostgresStore(load_settings().database_url) + await store.connect() + try: + rows = await store.pool.fetch('SELECT a.id, a.status, extract(epoch from (j.first_started_at-j.created_at))::float8 AS queue_wait_seconds FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.chat_id=$1', chat) + jobs = [dict(row, id=str(row['id'])) for row in rows] + pending = await store.pool.fetchval("SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')") + finally: + await store.close() + rss = 0 + for path in Path('/proc').glob('[0-9]*/status'): + try: + if int(path.parent.name) == os.getpid(): + continue + for line in path.read_text().splitlines(): + if line.startswith('VmRSS:'): + rss += int(line.split()[1]) * 1024 + except (OSError, ValueError): + pass + print(json.dumps({'jobs': jobs, 'pending': pending, 'rss_bytes': rss, + 'concurrency': int(os.environ['WORKER_CONCURRENCY']), + 'prefetch': int(os.environ['WORKER_PREFETCH'])})) + + +if __name__ == '__main__': + asyncio.run(main(UUID(sys.argv[1]))) diff --git a/src/reconcile.py b/src/reconcile.py index 47dfb5b..1b60bb8 100644 --- a/src/reconcile.py +++ b/src/reconcile.py @@ -9,6 +9,7 @@ import argparse import asyncio from datetime import datetime, timedelta, timezone import json +import os from time import perf_counter from .config import load_settings @@ -33,7 +34,11 @@ async def scan(args, db, s3) -> dict: # Age is measured at scan start, not after a potentially slow inventory. cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds) started = perf_counter() - objects, s3_pages = await s3.list_with_pages("originals/") + objects, s3_pages = {}, 0 + for prefix in ('staging/', 'originals/', 'variants/'): + entries, pages = await s3.list_with_pages(prefix) + objects.update(entries) + s3_pages += pages listed_at = perf_counter() metadata = await db.attachment_keys() metadata_at = perf_counter() @@ -74,15 +79,25 @@ async def scan(args, db, s3) -> dict: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--apply", action="store_true", help="delete only aged orphan objects") - parser.add_argument("--grace-seconds", type=int, default=3600) + parser.add_argument("--grace-seconds", type=int, default=int(os.environ.get('STAGING_RETENTION_SECONDS', '86400'))) + parser.add_argument('--loop', action='store_true') parser.add_argument("--object-key", action="append", default=[], help="restrict apply to these exact keys; repeatable, inventory stays complete") args = parser.parse_args() if not 0 <= args.grace_seconds <= 7 * 24 * 3600: parser.error("--grace-seconds must be 0..604800") - if any(not key.startswith("originals/") for key in args.object_key): - parser.error("--object-key must start with originals/") - print(json.dumps(asyncio.run(run(args)), ensure_ascii=False, sort_keys=True)) + if any(not key.startswith(('originals/', 'staging/', 'variants/')) for key in args.object_key): + parser.error('unsupported object prefix') + if args.loop: + import time + while True: + try: + print(json.dumps(asyncio.run(run(args)), ensure_ascii=False, sort_keys=True)) + except Exception: + print('cleanup deferred: dependency unavailable', flush=True) + time.sleep(60) + else: + print(json.dumps(asyncio.run(run(args)), ensure_ascii=False, sort_keys=True)) if __name__ == "__main__": diff --git a/src/render.py b/src/render.py new file mode 100644 index 0000000..dbb1d1d --- /dev/null +++ b/src/render.py @@ -0,0 +1,27 @@ +"""Isolated image renderer. Parent enforces wall time and kills this process.""" +import os +import pickle +import sys + + +def main(): + if sys.platform == 'linux': + import resource + limit = int(os.environ.get('RENDER_MEMORY_MB', '768')) * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) + from .worker import variants + from PIL import UnidentifiedImageError, Image + try: + data = sys.stdin.buffer.read(10485761) + if len(data) > 10485760: + raise ValueError('bytes limit') + result = variants(data, int(os.environ.get('MAX_IMAGE_PIXELS', '40000000'))) + pickle.dump(result, sys.stdout.buffer) + except MemoryError: + sys.exit(21) + except (ValueError, OSError, SyntaxError, UnidentifiedImageError, Image.DecompressionBombError): + sys.exit(20) + + +if __name__ == '__main__': + main() diff --git a/src/storage/postgres.py b/src/storage/postgres.py index f63699d..c06f389 100644 --- a/src/storage/postgres.py +++ b/src/storage/postgres.py @@ -247,6 +247,11 @@ class PostgresStore: ) if count != len(attachment_ids): raise ApiError(404, "not_found", "Вложение не найдено") + failed = await connection.fetchval( + "SELECT 1 FROM attachments WHERE id=ANY($1::uuid[]) AND status='failed' LIMIT 1", attachment_ids + ) + if failed: + raise ApiError(409, 'conflict', 'Вложение завершилось ошибкой') claimed = await connection.fetchrow( "INSERT INTO idempotency_records (user_id, method, path, idempotency_key, request_hash) " "VALUES ($1, 'POST', $2, $3, $4) ON CONFLICT DO NOTHING RETURNING message_id", @@ -396,16 +401,23 @@ class PostgresStore: attachment_id, ) await connection.execute( - "UPDATE attachment_jobs SET status = 'processing', lease_until = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1", + "UPDATE attachment_jobs SET status = 'processing', first_started_at=COALESCE(first_started_at, now()), lease_until = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1", attachment_id, lease_seconds, ) - return dict(row) + result = dict(row) + result['claim_time'] = await connection.fetchval('SELECT processing_started_at FROM attachments WHERE id=$1', attachment_id) + return result except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error - async def complete_attachment_job(self, attachment_id: UUID, original_key: str, variants: list[dict]) -> None: + async def complete_attachment_job(self, attachment_id: UUID, original_key: str, variants: list[dict], claim_time) -> bool: try: async with self.pool.acquire() as connection, connection.transaction(): + valid = await connection.fetchval( + "SELECT 1 FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id=$1 AND a.status='processing' AND a.processing_started_at=$2 AND j.lease_until > now() FOR UPDATE", attachment_id, claim_time + ) + if not valid: + return False for variant in variants: await connection.execute( "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) " @@ -418,28 +430,20 @@ class PostgresStore: attachment_id, original_key, ) await connection.execute("UPDATE attachment_jobs SET status = 'ready', lease_until = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id) + return True except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error - async def fail_attachment_job(self, attachment_id: UUID, code: str) -> None: - try: - await self.pool.execute( - "UPDATE attachments SET status = 'failed', error_code = $2, updated_at = now() WHERE id = $1", attachment_id, code - ) - await self.pool.execute( - "UPDATE attachment_jobs SET status = 'failed', last_error = $2, updated_at = now() WHERE attachment_id = $1", attachment_id, code - ) - except DB_ERRORS as error: - raise RuntimeError("PostgreSQL is unavailable") from error - - async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int, delay_seconds: float) -> bool: + async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int, delay_seconds: float, claim_time, permanent_code=None) -> bool: """Return true only when a bounded at-least-once retry was recorded.""" try: async with self.pool.acquire() as connection, connection.transaction(): job = await connection.fetchrow( - "SELECT attempt FROM attachment_jobs WHERE attachment_id = $1 FOR UPDATE", attachment_id + "SELECT j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.id=$1 AND a.status='processing' AND a.processing_started_at=$2 FOR UPDATE", attachment_id, claim_time ) - if job is not None and job["attempt"] < max_attempts: + if job is None: + return False + if permanent_code is None and job["attempt"] < max_attempts: await connection.execute( "UPDATE attachment_jobs SET attempt = attempt + 1, status = 'queued', last_error = 'processing_failed', lease_until = NULL, next_attempt_at = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1", attachment_id, delay_seconds, @@ -448,14 +452,16 @@ class PostgresStore: "UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", attachment_id ) return True - await self.fail_attachment_job(attachment_id, "retry_exhausted") - return False + code = permanent_code or 'retry_exhausted' + await connection.execute("UPDATE attachments SET status='failed', error_code=$2, updated_at=now() WHERE id=$1", attachment_id, code) + await connection.execute("UPDATE attachment_jobs SET status='failed', last_error=$2, lease_until=NULL, updated_at=now() WHERE attachment_id=$1", attachment_id, code) + return False except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error async def due_outbox_events(self, limit: int = 20) -> list[dict]: try: - return [dict(row) for row in await self.pool.fetch( + return [{**dict(row), 'payload': json.loads(row['payload']) if isinstance(row['payload'], str) else row['payload']} for row in await self.pool.fetch( "SELECT id, payload FROM attachment_outbox WHERE published_at IS NULL ORDER BY created_at LIMIT $1", limit )] except DB_ERRORS as error: @@ -464,7 +470,7 @@ class PostgresStore: async def mark_outbox_published(self, event_id: UUID) -> None: await self.pool.execute("UPDATE attachment_outbox SET published_at = now(), attempts = attempts + 1 WHERE id = $1", event_id) - async def schedule_due_jobs(self) -> None: + async def schedule_due_jobs(self, max_attempts: int = 5) -> None: """Outbox dispatcher also restores expired leases and scheduled retries.""" try: async with self.pool.acquire() as connection, connection.transaction(): @@ -474,9 +480,15 @@ class PostgresStore: "FOR UPDATE SKIP LOCKED" ) for row in rows: + row = dict(row) if row["status"] == "processing": + if row['attempt'] >= max_attempts: + await connection.execute("UPDATE attachment_jobs SET status='failed', lease_until=NULL, last_error='retry_exhausted', updated_at=now() WHERE job_id=$1", row['job_id']) + await connection.execute("UPDATE attachments SET status='failed', error_code='retry_exhausted', updated_at=now() WHERE id=$1", row['attachment_id']) + continue + row['attempt'] += 1 await connection.execute( - "UPDATE attachment_jobs SET status = 'queued', lease_until = NULL, next_attempt_at = now(), last_error = 'lease_expired', updated_at = now() WHERE job_id = $1", row["job_id"] + "UPDATE attachment_jobs SET status = 'queued', attempt=$2, lease_until = NULL, next_attempt_at = now(), last_error = 'lease_expired', updated_at = now() WHERE job_id = $1", row["job_id"], row['attempt'] ) await connection.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", row["attachment_id"]) payload = {"schema_version": 1, "job_id": str(row["job_id"]), "type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]), "pipeline_version": row["pipeline_version"], "attempt": row["attempt"], "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")} @@ -505,7 +517,7 @@ class PostgresStore: async def attachment_keys(self) -> dict[str, UUID]: """Inventory used only by the maintenance reconciler.""" try: - rows = await self.pool.fetch("SELECT COALESCE(original_key, staging_key) AS object_key, id FROM attachments") + rows = await self.pool.fetch("SELECT staging_key AS object_key, id FROM attachments WHERE status <> 'ready' UNION ALL SELECT original_key, id FROM attachments WHERE original_key IS NOT NULL UNION ALL SELECT object_key, attachment_id AS id FROM attachment_variants") except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error return {row["object_key"]: row["id"] for row in rows} diff --git a/src/worker.py b/src/worker.py index 3e7f2d5..47fb19e 100644 --- a/src/worker.py +++ b/src/worker.py @@ -6,8 +6,13 @@ import json import logging import os import random +import pickle +import subprocess +import sys +import time +from pathlib import Path from concurrent.futures import ThreadPoolExecutor -from uuid import UUID +from uuid import UUID, uuid4 from PIL import Image, ImageOps @@ -19,6 +24,32 @@ from .storage.s3 import S3Settings, S3Store logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s worker %(message)s") +class PermanentFailure(Exception): + pass + + +def fit(image, bound): + longest = max(image.size) + if longest <= bound: + return image.copy() + return image.resize((max(1, image.width * bound // longest), + max(1, image.height * bound // longest)), Image.Resampling.LANCZOS) + + +def render_isolated(data, timeout): + try: + result = subprocess.run([sys.executable, '-m', 'src.render'], input=data, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=timeout) + except subprocess.TimeoutExpired as error: + raise TimeoutError('render deadline exceeded') from error + if result.returncode in (20, 21): + raise PermanentFailure('invalid_image' if result.returncode == 20 else 'image_resource_limit') + if result.returncode: + raise RuntimeError('renderer exited') + # Only our own child process produces this payload; no external pickle input. + return pickle.loads(result.stdout) + + def thumbnail(data: bytes) -> tuple[bytes, int, int]: with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source: image = ImageOps.exif_transpose(source) @@ -28,7 +59,7 @@ def thumbnail(data: bytes) -> tuple[bytes, int, int]: image = background.convert("RGB") else: image = image.convert("RGB") - image.thumbnail((256, 256), Image.Resampling.LANCZOS) + image = fit(image, 256) output = BytesIO() image.save(output, format="JPEG", quality=85, optimize=True) return output.getvalue(), image.width, image.height @@ -44,8 +75,7 @@ def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes white.alpha_composite(image) rgb = white.convert("RGB") - thumb = rgb.copy() - thumb.thumbnail((256, 256), Image.Resampling.LANCZOS) + thumb = fit(rgb, 256) thumbnail_out = BytesIO(); thumb.save(thumbnail_out, "JPEG", quality=85, optimize=True) side = min(rgb.width, rgb.height) @@ -53,8 +83,7 @@ def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes crop = rgb.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS) crop_out = BytesIO(); crop.save(crop_out, "JPEG", quality=85, optimize=True) - optimized = rgb.copy() - optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS) + optimized = fit(rgb, 1280) optimized_out = BytesIO(); optimized.save(optimized_out, "WEBP", quality=80, method=4) return [ ("thumbnail", thumbnail_out.getvalue(), "image/jpeg", thumb.width, thumb.height), @@ -64,26 +93,50 @@ def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> None: + # Excludes cleanup for the whole lifetime of S3 effects, including stale attempts. + async with store.attachment_transaction(): + await process_locked(store, s3, payload, lease_seconds) + + +async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> None: attachment_id = UUID(payload["attachment_id"]) attachment = await store.claim_attachment_job(attachment_id, lease_seconds) if attachment is None: return - original_key = f"originals/{attachment_id}" + payload['_claim_time'] = attachment['claim_time'] + payload['attempt'] = attachment.get('attempt', payload.get('attempt', 1)) + attempt_key = uuid4().hex + original_key = f"originals/{attachment_id}/{attempt_key}" await s3.copy(attachment["staging_key"], original_key, attachment["content_type"]) original = await s3.download_key(attachment["staging_key"], attachment["size_bytes"], attachment["sha256"]) - rendered = await asyncio.wait_for( - asyncio.to_thread(variants, original, s3.settings.max_pixels), timeout=lease_seconds - ) + rendered = await asyncio.to_thread(render_isolated, original, lease_seconds) metadata = [] for name, content, content_type, width, height in rendered: extension = "webp" if name == "optimized" else "jpg" - object_key = f"variants/{attachment_id}/{name}.{extension}" + object_key = f"variants/{attachment_id}/{attempt_key}/{name}.{extension}" await s3.put(object_key, BytesIO(content), len(content), content_type) metadata.append({"name": name, "object_key": object_key, "content_type": content_type, "size_bytes": len(content), "width": width, "height": height, "sha256": sha256(content).hexdigest()}) - await store.complete_attachment_job(attachment_id, original_key, metadata) - await s3.delete(attachment["staging_key"]) + committed = await store.complete_attachment_job(attachment_id, original_key, metadata, attachment['claim_time']) + if committed: + try: + await s3.delete(attachment["staging_key"]) + except Exception: + logging.warning('staging cleanup deferred attachment_id=%s', attachment_id) + + +def settle_delivery(channel, delivery_tag, future): + """Only a successful durable disposition permits acknowledgement.""" + try: + success = future.result() is True + except Exception: + success = False + if channel.is_open: + if success: + channel.basic_ack(delivery_tag) + else: + channel.basic_nack(delivery_tag, requeue=True) def main() -> None: @@ -104,46 +157,61 @@ def main() -> None: s3.client.close() await store.close() - async def schedule_retry(attachment_id, delay): + async def schedule_retry(attachment_id, delay, claim_time, permanent_code=None): retry_store = PostgresStore(settings.database_url) await retry_store.connect() try: return await retry_store.retry_attachment_job( - attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay + attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay, claim_time, permanent_code ) finally: await retry_store.close() def handle(body): + payload = None try: payload = json.loads(body) + if not isinstance(payload, dict): + raise ValueError('job must be an object') if payload["type"] != "attachment.process.v1" or payload["schema_version"] != 1: raise ValueError("unexpected broker payload") + UUID(payload['attachment_id']) + UUID(payload['job_id']) + except (ValueError, KeyError, TypeError): + try: + asyncio.run(publish_dlq({'failure': 'invalid_job'})) + return True + except Exception: + return False + try: asyncio.run(handle_async(payload)) - except Exception: - logging.exception("job failed") + except Exception as error: + logging.warning("job failed attachment_id=%s", payload['attachment_id']) + if '_claim_time' not in payload: + return False try: attachment_id = UUID(payload["attachment_id"]) attempt = int(payload.get("attempt", 1)) delay = min(60.0, float(2 ** min(attempt, 6))) * random.uniform(0.5, 1.5) - retry = asyncio.run(schedule_retry(attachment_id, delay)) + asyncio.run(schedule_retry(attachment_id, delay, payload['_claim_time'], + str(error) if isinstance(error, PermanentFailure) else None)) except Exception: - retry = False - if not retry: - try: - asyncio.run(publish_dlq(payload)) - except Exception: - logging.exception("could not publish failed job to DLQ") - return None + return False + return True def consume(ch, method, properties, body): del properties future = executor.submit(handle, body) def acknowledged(_future): - connection.add_callback_threadsafe(lambda: ch.basic_ack(method.delivery_tag)) + if connection.is_open: + connection.add_callback_threadsafe(lambda: settle_delivery(ch, method.delivery_tag, _future)) future.add_done_callback(acknowledged) try: + def heartbeat(): + Path('/tmp/worker-heartbeat').write_text(str(time.time())) + connection.call_later(2, heartbeat) + heartbeat() with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="attachment-worker") as executor: channel.basic_consume(queue=QUEUE, on_message_callback=consume, auto_ack=False) channel.start_consuming() diff --git a/student-tests/attachments.py b/student-tests/attachments.py index 8165232..f9f7531 100644 --- a/student-tests/attachments.py +++ b/student-tests/attachments.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Attachment contract checks against the running lab4 stack.""" +"""Attachment contract checks against the running lab5 stack.""" import hashlib import http.client from io import BytesIO @@ -23,16 +23,17 @@ def main(): path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' png = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes() - def upload(data, mime='image/png', filename='photo.png', status=201, actor=alice): + def upload(data, mime='image/png', filename='photo.png', status=202, actor=alice): return c.call('POST', path + quote(filename, safe=''), status, actor=actor, raw=data, - headers={'Content-Type': mime}, schema='Attachment' if status == 201 else None) + headers={'Content-Type': mime}, schema='Attachment' if status == 202 else None) for fmt, mime in [('PNG', 'image/png'), ('JPEG', 'image/jpeg')]: buf = BytesIO() Image.new('RGB', (32, 20), 'blue').save(buf, fmt) data = buf.getvalue() att = upload(data, mime)['json'] - check(att['variants'] == [] and att['status'] == 'ready', 'wrong attachment state') + check(att['variants'] == [] and att['status'] == 'queued', 'wrong attachment state') + att = c.wait_ready(bob, att) check(att['sha256'] == hashlib.sha256(data).hexdigest(), 'wrong checksum') content_path = '/api/v1/attachments/' + att['id'] check(c.call('GET', content_path, actor=bob, schema='Attachment')['json'] == att, @@ -41,7 +42,7 @@ def main(): c.call('GET', content_path, 404, actor=eve) c.call('GET', content_path + '/content', 404, actor=eve) c.call('GET', content_path + '/content', 401) - c.call('GET', content_path + '/content?variant=thumbnail', 404, actor=bob) + c.call('GET', content_path + '/content?variant=thumbnail', actor=bob) message = c.message(bob, chat, '', attachments=[att['id']])['json'] history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=alice, schema='MessagePage')['json']['items'] @@ -98,6 +99,7 @@ def main(): # Exact limits are accepted; original bytes (including PNG trailing data) survive. exact = png + b'\0' * (10485760 - len(png)) att = upload(exact)['json'] + c.wait_ready(bob, att) check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == exact, '10 MiB original changed') buf = BytesIO() diff --git a/student-tests/pipeline_critical.py b/student-tests/pipeline_critical.py new file mode 100644 index 0000000..dfb716f --- /dev/null +++ b/student-tests/pipeline_critical.py @@ -0,0 +1,107 @@ +"""Offline regression checks for delivery and terminal-state safety.""" +import asyncio +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from io import BytesIO +import json +from pathlib import Path +import sys +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, MagicMock, patch +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.storage.postgres import PostgresStore +from src.worker import process_locked as process, settle_delivery +from concurrent.futures import Future +from src.broker import _publish_to + + +class CriticalTests(unittest.IsolatedAsyncioTestCase): + def test_failed_or_unrecorded_work_is_requeued(self): + for result in (False, None, RuntimeError('database unavailable')): + future, channel = Future(), MagicMock() + if isinstance(result, Exception): + future.set_exception(result) + else: + future.set_result(result) + settle_delivery(channel, 7, future) + channel.basic_ack.assert_not_called() + channel.basic_nack.assert_called_once_with(7, requeue=True) + + def test_durable_success_is_acked(self): + future, channel = Future(), MagicMock() + future.set_result(True) + settle_delivery(channel, 7, future) + channel.basic_ack.assert_called_once_with(7) + channel.basic_nack.assert_not_called() + + async def test_outbox_decodes_jsonb(self): + store = PostgresStore('unused') + store._pool = AsyncMock() + payload = {'type': 'attachment.process.v1', 'attachment_id': str(uuid4())} + store.pool.fetch.return_value = [{'id': uuid4(), 'payload': json.dumps(payload)}] + event, = await store.due_outbox_events() + self.assertEqual(json.loads(json.dumps(event['payload'])), payload) + + async def run_processing(self, committed): + store, s3 = AsyncMock(), AsyncMock() + stamp = datetime.now(timezone.utc) + store.claim_attachment_job.return_value = { + 'claim_time': stamp, 'staging_key': 'staging/test', + 'content_type': 'image/png', 'size_bytes': 1, 'sha256': 'unused', + } + store.complete_attachment_job.return_value = committed + s3.settings = SimpleNamespace(max_pixels=40000000) + s3.download_key.return_value = b'fixture' + s3.delete.side_effect = RuntimeError('storage temporarily unavailable') + payload = {'attachment_id': str(uuid4())} + with patch('src.worker.render_isolated', return_value=[('thumbnail', b'jpeg', 'image/jpeg', 1, 1)]): + await process(store, s3, payload, 60) + self.assertEqual(store.complete_attachment_job.call_args.args[-1], stamp) + return store, s3 + + async def test_cleanup_error_does_not_retry_ready(self): + store, s3 = await self.run_processing(True) + s3.delete.assert_awaited_once() + store.retry_attachment_job.assert_not_awaited() + + async def test_stale_completion_keeps_staging(self): + _, s3 = await self.run_processing(False) + s3.delete.assert_not_awaited() + + async def test_stale_claim_cannot_publish_metadata(self): + store = PostgresStore('unused') + connection = AsyncMock() + connection.fetchval.return_value = None + @asynccontextmanager + async def context(): + yield connection + connection.transaction = context + store._pool = SimpleNamespace(acquire=context) + self.assertFalse(await store.complete_attachment_job(uuid4(), 'original', [], datetime.now(timezone.utc))) + connection.execute.assert_not_awaited() + + async def test_retry_of_terminal_or_stale_attempt_is_noop(self): + store = PostgresStore('unused') + connection = AsyncMock() + connection.fetchrow.return_value = None + @asynccontextmanager + async def context(): + yield connection + connection.transaction = context + store._pool = SimpleNamespace(acquire=context) + self.assertFalse(await store.retry_attachment_job(uuid4(), 5, 1, datetime.now(timezone.utc))) + connection.execute.assert_not_awaited() + + def test_dlq_waits_for_publisher_confirm(self): + connection = MagicMock() + with patch('src.broker._connection', return_value=connection): + _publish_to('dlq', {'failure': 'invalid_job'}) + connection.channel.return_value.confirm_delivery.assert_called_once() + self.assertTrue(connection.channel.return_value.basic_publish.call_args.kwargs['mandatory']) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/remaining_regressions.py b/student-tests/remaining_regressions.py new file mode 100644 index 0000000..af79f76 --- /dev/null +++ b/student-tests/remaining_regressions.py @@ -0,0 +1,127 @@ +"""Contract, isolated-render and benchmark regressions without Docker.""" +import asyncio +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +import importlib.util +from io import BytesIO +import json +from pathlib import Path +import sys +import tempfile +import threading +import time +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from PIL import Image +from src.worker import variants, render_isolated, PermanentFailure +from src.storage.postgres import PostgresStore +from src.reconcile import scan +from src.errors import ApiError + + +def png(width, height): + buffer = BytesIO() + Image.new('RGBA', (width, height), (255, 0, 0, 0)).save(buffer, 'PNG') + return buffer.getvalue() + + +class Regressions(unittest.IsolatedAsyncioTestCase): + async def test_failed_attachment_is_rejected(self): + store, connection = PostgresStore('unused'), AsyncMock() + connection.fetchval.side_effect = [1, 1, 1] + @asynccontextmanager + async def context(): + yield connection + connection.transaction = context + store._pool = SimpleNamespace(acquire=context) + with self.assertRaises(ApiError) as error: + await store.create_message_idempotent(uuid4(), uuid4(), '', 'key', 86400, [uuid4()]) + self.assertEqual(error.exception.status_code, 409) + connection.fetchrow.assert_not_awaited() + + def test_floor_and_small_images(self): + self.assertEqual(variants(png(300, 200))[0][3:], (256, 170)) + self.assertEqual(variants(png(200, 300))[0][3:], (170, 256)) + self.assertEqual(variants(png(10, 8))[0][3:], (10, 8)) + for name, data, mime, width, height in variants(png(300, 200)): + with Image.open(BytesIO(data)) as image: + self.assertEqual(image.size, (width, height)) + self.assertTrue(all(value > 240 for value in image.convert('RGB').getpixel((0, 0)))) + + def test_renderer_success_and_permanent_error(self): + self.assertEqual(render_isolated(png(300, 200), 10)[0][3:], (256, 170)) + with self.assertRaises(PermanentFailure): + render_isolated(b'not an image', 10) + + def test_renderer_is_killed_on_deadline(self): + started = time.monotonic() + with self.assertRaises(TimeoutError): + render_isolated(png(300, 200), .0001) + self.assertLess(time.monotonic() - started, 2) + + async def test_cleanup_retains_active_and_failed_staging(self): + old = datetime.now(timezone.utc) - timedelta(days=2) + db, s3 = AsyncMock(), AsyncMock() + db.attachment_keys.return_value = {'staging/active': uuid4(), 'staging/failed': uuid4(), 'originals/ready': uuid4()} + s3.list_with_pages.side_effect = [({'staging/active': old, 'staging/failed': old, 'staging/orphan': old}, 1), + ({'originals/ready': old}, 1), ({'variants/orphan': old}, 1)] + report = await scan(SimpleNamespace(grace_seconds=60, apply=True, object_key=[]), db, s3) + self.assertEqual(report['deleted'], ['staging/orphan', 'variants/orphan']) + + async def test_permanent_failure_is_atomic_and_not_retried(self): + store = PostgresStore('unused') + connection = AsyncMock() + connection.fetchrow.return_value = {'attempt': 1} + @asynccontextmanager + async def context(): + yield connection + connection.transaction = context + store._pool = SimpleNamespace(acquire=context) + result = await store.retry_attachment_job(uuid4(), 5, 1, datetime.now(timezone.utc), 'invalid_image') + self.assertFalse(result) + self.assertEqual(connection.execute.await_count, 2) + for call in connection.execute.call_args_list: + self.assertIn("status='failed'", call.args[0]) + self.assertEqual(call.args[2], 'invalid_image') + + def test_benchmark_partial_batch_and_recovery(self): + spec = importlib.util.spec_from_file_location('track1', ROOT / 'scripts/track1.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + jobs, lock = {}, threading.Lock() + class FakeClient: + def __init__(self, *args, **kwargs): pass + def bootstrap(self): return {}, {}, {}, {'id': str(uuid4())} + def call(self, method, path, *args, **kwargs): + if method == 'POST': + key = str(uuid4()) + with lock: jobs[key] = {'id': key, 'status': 'ready', 'queue_wait_seconds': .01} + return {'status': 202, 'json': {'id': key}} + return {'json': {'status': 'ready'}} + def attachment(self, *args): return {}, b'' + def wait_ready(self, *args, **kwargs): return {} + def metrics(_): + with lock: + return {'jobs': list(jobs.values()), 'pending': 0, 'rss_bytes': 1000, 'concurrency': 2, 'prefetch': 2} + with tempfile.TemporaryDirectory() as directory: + destination = str(Path(directory) / 'result.json') + args = SimpleNamespace(run_name='test', workers=2, prefetch=2, max_pending=20, images=10, + concurrency=8, size=8, max_ready_seconds=2, require_backpressure=False, + output=destination, base_url=None, ca_file=None) + info = {'Config': {'Env': ['MAX_PENDING_JOBS=20']}, 'HostConfig': {'Memory': 4096, 'NanoCpus': 2}} + with patch.object(module, 'arguments', return_value=args), patch.object(module, 'Client', FakeClient), \ + patch.object(module, 'metrics', side_effect=metrics), patch.object(module, 'container_info', return_value=info), \ + patch.object(module, 'broker_depth', return_value={'messages_ready': 0, 'messages_unacknowledged': 0}): + module.main() + report = json.loads(Path(destination).read_text()) + self.assertEqual(report['results']['ready'], 10) + self.assertTrue(report['results']['recovery_upload_ready']) + + +if __name__ == '__main__': + unittest.main()