From b6953ae6eed1ebd340fe81ad624390cd337a00c9 Mon Sep 17 00:00:00 2001 From: Mikhail Verkovykh Date: Fri, 20 Nov 2026 19:05:29 +0300 Subject: [PATCH] feat(lab5): add exactly_right scenario and refresh evidence --- CHANGELOG.md | 17 +- README.md | 29 +- REPORT.md | 133 +- evidence/track1.html | 273 ++-- evidence/track1.json | 2505 ++++++++++++++++++++++++++++------ evidence/track1.md | 23 +- scripts/track1_experiment.py | 224 ++- 7 files changed, 2529 insertions(+), 675 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 523985d3..1415f9de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,14 +26,27 @@ для smoke, отказов, persistence и failover. - `scripts/restart.sh` перезапускает RabbitMQ и worker с сохранением volumes. - `scripts/track1_experiment.py` и `make track1` — трек 1: матрица - `serial/balanced/eager` × `small_many`/`huge_few`, throughput, queue lag, - p95 time-to-ready, RSS и отдельный backpressure/recovery прогон; + `serial/balanced/eager` × `small_many`/`exactly_right`/`huge_few`, + throughput, queue lag, p95 time-to-ready, RSS, декомпозиция + decode/scale/JPEG/WebP и отдельный backpressure/recovery прогон; self-contained HTML-дашборд с inline SVG. +- `exactly_right` (1280×1280): optimized WebP кодируется без scaling, поэтому + encode share показывает нижнюю границу стоимости сжатия. - Документация: README с запуском и архитектурой, отчёт с таблицей критериев, CHANGELOG. +### Removed + +- 34 устаревших файла `*.lab3`/`*.lab4` (Makefile, contracts, tests, scripts, + CI templates и fixtures) удалены из рабочего дерева. + ### Changed +- Параметры вариантов (`ThumbnailMax`, `CropSize`, `OptimizedMax`, качества + JPEG/WebP и WebP method) перенесены в `app/internal/config/config.go`. +- Комментарии в Go/SQL/shell/Python больше не ссылаются на номера лабораторных; + поведение описывается по существу. + - `POST /chats/{id}/attachments` отвечает `202` после durable staging и COMMIT задания; готовность определяется опросом метаданных, content до `ready` — 409. - Загрузка потоковая (буфер 32 KiB), лимиты bytes/pixels проверяются до diff --git a/README.md b/README.md index bc3f00c6..402bbf64 100644 --- a/README.md +++ b/README.md @@ -150,35 +150,36 @@ python3 scripts/track1_experiment.py # трек 1 уров `make test-persistence` / `make test-failover` с `scripts/restart.sh` и `scripts/stop-one.sh`; см. `evidence/lab5-persistence*` и `evidence/lab5-failover.txt`. -Трек 1 запускает матрицу профилей `serial/balanced/eager` на маленьких и -больших изображениях, измеряет throughput, queue lag, p95 time-to-ready и RSS, -затем отдельно показывает backpressure и восстановление. Результаты и графики — +Трек 1 запускает матрицу `serial/balanced/eager` на трёх размерах: +`small_many` (192×128), `exactly_right` (1280×1280, optimized WebP без scaling) +и `huge_few` (2400×1800). Метрики: throughput, queue lag, p95 time-to-ready, +RSS и разложение времени на decode / scale / JPEG / WebP. Результаты и графики — `evidence/track1.{json,html,md}`, разбор — в [REPORT.md](REPORT.md). ```sh -# Полный прогон: 30 small + 12 huge на каждый профиль, затем backpressure. +# Полный прогон: 30 small + 18 exactly_right + 12 huge на профиль. make track1 BASE_URL=http://localhost:8080 # То же без make. python3 scripts/track1_experiment.py --base-url http://localhost:8080 # Быстрая проверка механики и/или свои лимиты worker-а. -python3 scripts/track1_experiment.py --small-jobs 5 --huge-jobs 3 --skip-backpressure +python3 scripts/track1_experiment.py --small-jobs 5 --exactly-jobs 3 --huge-jobs 3 --skip-backpressure TRACK1_WORKER_CPU_LIMIT=2.0 TRACK1_WORKER_MEM_LIMIT=3G \ python3 scripts/track1_experiment.py --base-url http://localhost:8080 ``` Эксперимент пересоздаёт worker с `WORKER_CONCURRENCY`/`WORKER_PREFETCH`, -поднимает memory limit до 2 GiB (чтобы `eager` не падал по OOM на больших -изображениях), а в `finally` возвращает конфигурацию из `.env`. Он управляет -только локальным Compose-стендом и не предназначен для production. +поднимает memory limit до 2 GiB (чтобы `eager` не падал по OOM), а в `finally` +возвращает конфигурацию из `.env`. Он управляет только локальным Compose-стендом. -Кратко: на 192×128 eager дал 13.86 jobs/s против 6.88 у serial (2×) при RSS -28→33 MiB. На 2400×1800 все профили дали ~0.45 jobs/s (worker ограничен 1 CPU), -но RSS вырос 205→1122 MiB, а queue lag p95 упал 23.3→17.5 s: concurrency меняет -latency/ресурсы, а не throughput CPU-bound задач. При `MAX_PENDING_JOBS=3` -четвёртый upload получил `503` + `Retry-After: 5`; после старта worker три -принятых задания стали ready за 7.6 s. +Кратко: на 192×128 eager дал 15.58 jobs/s против 6.84 у serial (2.3×) при RSS +28→32 MiB. На 1280×1280 и 2400×1800 все профили дали ~0.44–0.50 jobs/s: +worker ограничен 1 CPU, а encode share составил 82–83% и 61–64% — сжатие, а не +только scaling, задаёт нижнюю границу. RSS вырос до 818–950 MiB, queue lag p95 +упал, но ready p95 остался ~26–36 s. При `MAX_PENDING_JOBS=3` API вернул +`503` + `Retry-After: 5`; после старта worker принятые задания стали ready за +7.65 s. ## Ограничения diff --git a/REPORT.md b/REPORT.md index ed8d1404..0921588c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -115,7 +115,7 @@ backoff ±50% jitter до `JOB_MAX_ATTEMPTS`. После исчерпания | Очистка staging после фиксации | сценарии `worker_stopped`, `crash_after_effect` | `staging_cleared: true`; reconciler удаляет только брошенное | `evidence/lab5-grade4.json`, `evidence/reconcile.txt` | | Негативные случаи и лимиты | `python3 tests/custom/attachments.py` | 413 по bytes и по 42 MP, 415 corrupt/spoof, 404/400, нет 5xx | `evidence/lab5-negative-cases.txt` | | Failover API | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | 118 успешных записей, 1 переходная ошибка, восстановление 2.659 s; остановлена именно `api-2` | `evidence/lab5-failover.txt` | -| Трек 1, уровень 5 | `python3 scripts/track1_experiment.py` | 3 профиля × 2 размера; eager/small 13.86 jobs/s против serial 6.88; huge_few ~0.45 jobs/s у всех, RSS 205→1122 MiB; cap=3 дал 503 + Retry-After 5 и восстановление за 7.596 s | `evidence/track1.json`, `evidence/track1.html` | +| Трек 1, уровень 5 | `python3 scripts/track1_experiment.py` | 3 профиля × 3 размера; eager/small 15.58 jobs/s против serial 6.84; exactly_right/huge_few ~0.44–0.50 jobs/s, encode share 83% против 61–64%; RSS до 950 MiB; 503 + Retry-After 5, recovery 7.653 s | `evidence/track1.json`, `evidence/track1.html` | ## Эксперименты и отказы @@ -143,42 +143,32 @@ HAProxy; отказ БД или брокера в этом сценарии не ### Гипотеза -Маленькие изображения, которые не требуют масштабирования, дешевле на задание: -выше throughput и ниже RSS. Большие изображения (2400×1800) требуют декодирования -и downscale до 1280×960, поэтому очередь растёт, p95 time-to-ready увеличивается, -а память сильнее зависит от concurrency. Дополнительно проверялся компромисс -«latency против ресурсов»: больше concurrency/prefetch должно снижать queue lag, -но повышать RSS. +Стоимость задания = decode + scaling + encode. `exactly_right` (1280×1280) не +масштабирует optimized WebP и показывает нижнюю границу стоимости сжатия; +`huge_few` добавляет downscale. Ожидаем: на тяжёлых изображениях concurrency +снижает queue lag, но не throughput при 1 CPU, а RSS растёт. ### Методика -- Профили: `serial` 1/1, `balanced` 4/4, `eager` 8/32 - (concurrency/prefetch). -- Сценарии: `small_many` — 30 PNG 192×128 (941 B, варианты не масштабируются); - `huge_few` — 12 PNG 2400×1800 (53 135 B, 4.32 MP, optimized уменьшается до - 1280×960). -- Для каждой из 6 ячеек worker пересоздавался с профилем, поэтому RSS не - переносился из предыдущего прогона. Изображения загружались параллельно - (до 8 HTTP-клиентов). Все задания дожидались терминального состояния. -- Метрики берутся из PostgreSQL по `attachment_jobs`: queue lag = - `processing_started_at - created_at`, time-to-ready = - `finished_at - created_at`, processing = `finished_at - processing_started_at`. - Throughput = число ready / (последний `finished_at` − первый `created_at`). -- RSS — `VmRSS` процесса worker из `/proc/1/status`, сэмпл примерно каждые - 100 мс; baseline снимался до загрузки. -- Worker ограничен **1 CPU**. На время эксперимента memory limit поднят до - **2 GiB** (в `.env` по умолчанию 1 GiB), иначе `eager/huge_few` упирался бы в - OOM, а не в измеряемый компромисс. Это зафиксировано в JSON. -- Backpressure: API пересоздан с `MAX_PENDING_JOBS=3`, worker остановлен, - upload-ы идут последовательно до первого `503`; затем worker запускается и - проверяется восстановление принятых заданий и новый probe upload. +- Профили: `serial` 1/1, `balanced` 4/4, `eager` 8/32 (concurrency/prefetch). +- Сценарии: `small_many` 30×192×128; `exactly_right` 18×1280×1280 (optimized + WebP без scaling); `huge_few` 12×2400×1800 (optimized → 1280×960). +- 9 ячеек; worker пересоздаётся на профиль, изображения загружаются параллельно, + все задания доводятся до терминального состояния. +- Queue lag, time-to-ready и processing — из `attachment_jobs`; throughput = + ready / (последний `finished_at` − первый `created_at`). +- Decode/scale/JPEG/WebP — из worker-лога; encode share = + `encode / (decode + scale + encode)`. Это wall-clock под данным профилем. +- RSS — `VmRSS` из `/proc/1/status` примерно каждые 100 мс. +- Worker 1 CPU, memory limit 2 GiB (в `.env` 1 GiB), чтобы `eager` не падал по + OOM; зафиксировано в JSON. +- Backpressure: `MAX_PENDING_JOBS=3`, worker остановлен, upload-ы до `503`, затем + recovery. Cap выбран намеренно; важны `Retry-After` и отсутствие потерь. Команда: ```sh make track1 BASE_URL=http://localhost:8080 -# или -python3 scripts/track1_experiment.py --base-url http://localhost:8080 ``` Сырые данные: `evidence/track1.json`; графики: `evidence/track1.html`; @@ -186,63 +176,46 @@ python3 scripts/track1_experiment.py --base-url http://localhost:8080 ### Результаты -| Профиль | Сценарий | Jobs/s | Queue lag p95, ms | Ready p95, ms | Processing p95, ms | RSS peak, MiB | Max parallel | -| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| serial | small_many | 6.875 | 3 194 | 3 284 | 113 | 28.2 | 1 | -| serial | huge_few | 0.443 | 23 333 | 25 521 | 2 308 | 204.7 | 1 | -| balanced | small_many | 11.590 | 1 624 | 1 736 | 209 | 30.6 | 4 | -| balanced | huge_few | 0.454 | 17 716 | 25 785 | 25 374 | 728.7 | 4 | -| eager | small_many | 13.858 | 898 | 1 158 | 284 | 32.7 | 8 | -| eager | huge_few | 0.459 | 17 538 | 25 699 | 24 979 | 1 122.1 | 8 | +| Профиль | Сценарий | Jobs/s | Queue lag p95, ms | Ready p95, ms | Processing p95, ms | Encode p95, ms | Encode share, % | RSS peak, MiB | Max parallel | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| serial | small_many | 6.837 | 3 256 | 3 338 | 170 | 80 | 83.8 | 28.2 | 1 | +| serial | exactly_right | 0.475 | 33 591 | 35 714 | 2 180 | 1 722 | 82.0 | 174.3 | 1 | +| serial | huge_few | 0.436 | 23 668 | 25 896 | 2 359 | 1 508 | 63.8 | 212.9 | 1 | +| balanced | small_many | 13.694 | 1 147 | 1 274 | 396 | 90 | 83.8 | 28.9 | 4 | +| balanced | exactly_right | 0.495 | 31 510 | 35 764 | 8 174 | 6 562 | 83.1 | 433.8 | 4 | +| balanced | huge_few | 0.441 | 18 161 | 26 576 | 9 046 | 5 606 | 61.4 | 756.4 | 4 | +| eager | small_many | 15.576 | 924 | 1 166 | 410 | 95 | 79.9 | 32.1 | 8 | +| eager | exactly_right | 0.495 | 31 735 | 35 797 | 16 363 | 13 330 | 83.1 | 818.4 | 8 | +| eager | huge_few | 0.443 | 17 648 | 26 641 | 17 678 | 11 124 | 61.4 | 949.6 | 8 | -Backpressure: cap `3`; первые 3 upload-а получили `202`, четвёртый — `503` с -`Retry-After: 5` и `error_code=unavailable`. После старта worker три принятых -задания стали `ready` за **7.596 s** (включая запуск контейнера и healthcheck), -после чего probe upload получил `202` и дошёл до `ready`. +Backpressure: cap `3` задан намеренно; первый `503` пришёл с `Retry-After: 5`. +После старта worker принятые задания стали `ready` за **7.653 s** (включая +запуск контейнера и healthcheck), probe upload получил `202` и дошёл до `ready`. ### Выводы -1. **Размер изображения важнее числа заданий.** При `balanced` маленькие файлы - дают 11.59 jobs/s, большие — 0.454 jobs/s (в 25.5× меньше); p95 time-to-ready - растёт с 1 736 до 25 785 ms (в 14.9×). Processing p50 для huge — 8 460 ms - против 211 ms для small: доминирует декодирование/масштабирование, а не - ожидание в очереди. -2. **Eager хорошо работает на маленьких заданиях.** 13.86 jobs/s против 6.88 у - serial (2.0×), queue lag p95 падает с 3 194 до 898 ms, RSS растёт всего с - 28.2 до 32.7 MiB. Это ожидаемый выигрыш от перекрытия лёгких задач. -3. **На huge_few concurrency не даёт throughput.** Все три профиля дают - ~0.44–0.46 jobs/s: worker упирается в 1 CPU, а декодирование и WebP-кодирование - уже насыщают его. Queue lag p95 снижается (23.3 → 17.5 s), но p95 - time-to-ready остаётся ~25.5 s: последние задания всё равно ждут своей доли - CPU. При этом RSS растёт с 205 MiB (serial) до 729 MiB (balanced) и - 1 122 MiB (eager). **Больше concurrency здесь — не бесплатное ускорение, а - рост памяти при том же throughput.** -4. **Backpressure ограничивает принятую работу.** При cap=3 API перестал - принимать новые файлы до чтения тела и вернул retryable `503`; уже - подтверждённые задания не потерялись и завершились после возврата worker. - Это и есть требуемое «bounded acceptance + recovery». -5. **Практический выбор.** Для лёгких изображений разумен `balanced`/`eager`; - для тяжёлых при одном CPU лучше `balanced` (или `serial`, если важна память): - throughput тот же, а RSS в 1.5–5.5× ниже, чем у eager. `prefetch` больше - `concurrency` лишь держит сообщения у worker-а и не увеличивает параллельную - обработку. +1. **Compression, not scaling, is the floor.** `exactly_right` даёт 82–83% + encode share и ~0.48 jobs/s; `huge_few` — 61–64% и ~0.44 jobs/s. Убрать + downscale недостаточно, чтобы 1280×1280 стал дешёвым: доминирует WebP. + На `huge_few` scale p95 выше (balanced: 3.4 s против 1.5 s), но optimized + output меньше (1.23 MP против 1.64 MP), поэтому throughput почти совпадает. +2. **Маленькие изображения масштабируются concurrency.** 6.84 → 13.69 → + 15.58 jobs/s, RSS 28 → 32 MiB. +3. **Тяжёлые изображения CPU-bound.** ~0.44–0.50 jobs/s у всех профилей; + queue lag падает, ready p95 держится ~26–36 s, RSS растёт 174 → 818 MiB + (`exactly_right`) и 213 → 950 MiB (`huge_few`). Concurrency покупает latency + ценой памяти, а не throughput. +4. **Backpressure.** `503` + `Retry-After: 5`; принятые задания восстановились + за 7.65 s. +5. **Практический выбор.** `balanced` для тяжёлых, `eager` для лёгких. -### Ограничения эксперимента +### Ограничения -- Один worker, один RabbitMQ, один PostgreSQL; это сравнение настроек, не HA и - не горизонтальное масштабирование. -- Worker ограничен 1 CPU и 2 GiB; на другом железе абсолютные числа и точка - насыщения будут другими. Относительный вывод «CPU-bound huge не ускоряется - concurrency» получен именно для этого лимита. -- PNG-фикстуры синтетические и хорошо сжимаются; доминирует стоимость - декодирования/масштабирования, а не сеть. Размеры в байтах и SHA-256 есть в - JSON. -- Один прогон на ячейку, без длительного прогрева и повторений; p95 по 12 - заданиям чувствителен к выбросу. Это учебная оценка, не SLA. -- `MAX_PENDING_JOBS` проверяется приблизительно и не одной транзакцией с - вставкой; при нескольких API-репликах возможен небольшой перебор. -- Время восстановления backpressure включает `docker compose start` и - healthcheck, поэтому это верхняя оценка, а не время чистой обработки. +- `exactly_right` кодирует 1.64 MP WebP, `huge_few` после downscale — ~1.23 MP: + сценарий убирает scaling, но не уравнивает объём encode. +- PNG-фикстуры синтетические; один прогон на ячейку, p95 по 12–18 заданиям + чувствителен к выбросу. +- Время recovery включает `docker compose start` и healthcheck. ## Ограничения и следующие шаги diff --git a/evidence/track1.html b/evidence/track1.html index 53e7638b..ada26941 100644 --- a/evidence/track1.html +++ b/evidence/track1.html @@ -49,18 +49,18 @@ code { background: #f1f5f9; padding: 1px 5px; border-radius: 4px; }

Lab5, трек 1: очередь под перегрузкой

-

Сгенерировано 2026-11-14T21:42:03.122104+00:00. Стенд: http://172.18.0.1:8080, +

Сгенерировано 2026-11-20T16:02:07.856909+00:00. Стенд: http://172.18.0.1:8080, 4 CPU, Docker 5.6.1.

-Гипотеза. Маленькие изображения, которые почти не требуют -масштабирования, должны давать высокий throughput и низкий RSS на задание. -Большие изображения требуют декодирования и downscale, поэтому каждое задание -дороже: при том же числе worker-ов очередь растёт, p95 time-to-ready -увеличивается, а пиковый RSS зависит от concurrency сильнее, чем у маленьких -файлов. Одновременно проверяем компромисс: eager уменьшает ожидание, -но повышает потребление памяти; serial экономит память, но ограничивает -throughput. +Гипотеза. Стоимость задания складывается из decode, масштабирования +и сжатия. exactly_right (1280×1280) не масштабирует optimized-вариант, +поэтому показывает нижнюю границу стоимости сжатия; huge_few добавляет +downscale, а small_many почти не масштабируется, но и пикселей в нём +мало. Большие изображения при том же числе worker-ов увеличивают очередь, +p95 time-to-ready и RSS. Одновременно проверяем компромисс: eager +уменьшает ожидание, но повышает память; serial экономит память, но +ограничивает throughput.
@@ -72,16 +72,15 @@ throughput.

Сценарии: -small_many — 192×128, 30 jobs, many small images; variants fit or crop without large scaling; huge_few — 2400×1800, 12 jobs, few 4.32 MP images; optimized variant must scale down to 1280.

+small_many — 192×128, 30 jobs, many small images; almost no scaling; exactly_right — 1280×1280, 18 jobs, largest variant encodes at native 1280, so no scaling on the WebP path; huge_few — 2400×1800, 12 jobs, few 4.32 MP images; optimized variant must scale down to 1280.

Ключевые числа

-
6
прогонов матрицы
-
1122 MiB
максимальный RSS worker
-
3
принято до первого 503
-
7.6 s
восстановление после backpressure
+
9
прогонов матрицы
+
950 MiB
максимальный RSS worker
+
7.7 s
восстановление после backpressure
@@ -91,27 +90,34 @@ small_many — 192×128, 30 jobs, many small images; variants fit or crop withou 0 jobs/s -4 jobs/s +5 jobs/s -8 jobs/s +9 jobs/s -12 jobs/s +14 jobs/s -16 jobs/s - -6.88 jobs/s - -11.59 jobs/s - -13.86 jobs/s -small_many - -0.44 jobs/s - -0.45 jobs/s - -0.46 jobs/s -huge_few +18 jobs/s + +6.84 jobs/s + +13.69 jobs/s + +15.58 jobs/s +small_many + +0.47 jobs/s + +0.49 jobs/s + +0.49 jobs/s +exactly_right + +0.44 jobs/s + +0.44 jobs/s + +0.44 jobs/s +huge_few serial @@ -126,27 +132,34 @@ small_many — 192×128, 30 jobs, many small images; variants fit or crop withou 0 ms -7606 ms +10560 ms -15213 ms +21120 ms -22819 ms +31680 ms -30426 ms - -3284 ms - -1736 ms - -1158 ms -small_many - -25521 ms - -25785 ms - -25699 ms -huge_few +42240 ms + +3338 ms + +1274 ms + +1166 ms +small_many + +35714 ms + +35764 ms + +35797 ms +exactly_right + +25896 ms + +26576 ms + +26641 ms +huge_few serial @@ -161,27 +174,34 @@ small_many — 192×128, 30 jobs, many small images; variants fit or crop withou 0 ms -6883 ms +9909 ms -13766 ms +19819 ms -20650 ms +29728 ms -27533 ms - -3194 ms - -1624 ms - -898 ms -small_many - -23333 ms - -17716 ms - -17538 ms -huge_few +39637 ms + +3256 ms + +1147 ms + +924 ms +small_many + +33591 ms + +31510 ms + +31735 ms +exactly_right + +23668 ms + +18161 ms + +17648 ms +huge_few serial @@ -196,27 +216,76 @@ small_many — 192×128, 30 jobs, many small images; variants fit or crop withou 0 MiB -331 MiB +280 MiB -662 MiB +560 MiB -993 MiB +840 MiB -1324 MiB - -28 MiB - -31 MiB - -33 MiB -small_many - -205 MiB - -729 MiB - -1122 MiB -huge_few +1121 MiB + +28 MiB + +29 MiB + +32 MiB +small_many + +174 MiB + +434 MiB + +818 MiB +exactly_right + +213 MiB + +756 MiB + +950 MiB +huge_few + +serial + +balanced + +eager +lower is better + +
+Compression overhead: доля encode в pipeline CPU +encode / (decode + scale + encode), % (меньше — дешевле сжатие). + +0 % + +25 % + +49 % + +74 % + +99 % + +83.8 % + +83.8 % + +79.9 % +small_many + +82.0 % + +83.1 % + +83.1 % +exactly_right + +63.8 % + +61.4 % + +61.4 % +huge_few serial @@ -229,11 +298,16 @@ small_many — 192×128, 30 jobs, many small images; variants fit or crop withou

Полная матрица

- - + +
ПрофильСценарийJobsJobs/sQueue lag p95, msReady p95, msProcessing p95, msRSS peak, MiBMax parallelMax backlog
serialsmall_many306.883194328415228127
serialhuge_few120.4423333255212311205112
balancedsmall_many3011.591624173633631426
balancedhuge_few120.4517716257858710729412
eagersmall_many3013.86898115856633826
eagerhuge_few120.461753825699173201122812
ПрофильСценарийJobsJobs/sQueue lag p95, msReady p95, msProcessing p95, msEncode p95, msEncode share, %RSS peak, MiBMax parallelMax backlog
serialsmall_many306.84325633381708083.828127
serialexactly_right180.4733591357142180172282.0174118
serialhuge_few120.4423668258962359150863.8213112
balancedsmall_many3013.69114712743969083.829424
balancedexactly_right180.4931510357648174656283.1434418
balancedhuge_few120.4418161265769046560661.4756412
eagersmall_many3015.5892411664109579.932822
eagerexactly_right180.493173535797163631333083.1818818
eagerhuge_few120.441764826641176781112461.4950812

Queue lag = processing_started_at - created_at. Time-to-ready = finished_at - created_at. Processing = finished_at - processing_started_at. +Encode = JPEG thumbnail + JPEG crop + WebP optimized; encode share = +encode / (decode + scale + encode) из worker-лога. +Суммы байт вариантов и оригинала остаются в JSON, но не выносятся в заголовок: +синтетический PNG сжимается слишком хорошо, чтобы отношение байт было честной +метрикой эффективности кодека. RSS — VmRSS процесса worker, сэмплирование примерно каждые 100 мс. Max parallel — максимум одновременно обрабатываемых заданий; max backlog — максимум принятых, но ещё не готовых заданий.

@@ -242,20 +316,22 @@ Max parallel — максимум одновременно обрабатыва

Backpressure и восстановление

Перед этим прогоном API пересоздан с MAX_PENDING_JOBS=3, worker -остановлен. Последовательные upload-ы показывают, где заканчивается принятая -работа. Затем worker запускается, принятые задания должны дойти до ready.

+остановлен. Cap выбран намеренно: важно не число принятых заданий, а то, что +API возвращает 503 с Retry-After и не теряет уже +подтверждённую работу. Затем worker запускается, принятые задания должны дойти +до ready.

- +
#HTTPLatency, msRetry-AfterError code
120218.2——
220217.4——
320210.8——
45033.95unavailable
120226.7——
220220.3——
320213.8——
45032.45unavailable

Recovery: 3 принятых заданий -стали ready за 7.60 с после старта worker. +стали ready за 7.65 с после старта worker. Финальный probe upload: HTTP 202 → ready.

Что это значит

-
  • small_many: eager быстрее serial в 2.0× по throughput.
  • small_many: пиковый RSS eager относится к serial как 1.2×.
  • huge_few: throughput eager и serial практически одинаков.
  • huge_few: пиковый RSS eager относится к serial как 5.5×.
  • При balanced p95 time-to-ready для huge_few в 14.9× больше, чем для small_many: размер изображения меняет стоимость задания, а не только число заданий.
  • На huge_few eager почти не поднял throughput (1.04×): worker ограничен 1.0 CPU, а декодирование и масштабирование упираются в CPU. Concurrency снижает queue lag, но при этом растёт RSS; это и есть компромисс latency/ресурсы, а не бесплатное ускорение.
  • serial ограничивает одновременно обрабатываемые задания одним, поэтому queue lag растёт, но RSS остаётся минимальным. eager держит больше заданий в работе и в prefetch-окне брокера.
  • Backpressure: при MAX_PENDING_JOBS=3 API принял 3 задания, затем вернул 503 с Retry-After=5; после старта worker очередь восстановилась за 7.6 с.
+
  • small_many: eager быстрее serial в 2.3× по throughput.
  • small_many: пиковый RSS eager относится к serial как 1.1×.
  • exactly_right: throughput eager и serial практически одинаков.
  • exactly_right: пиковый RSS eager относится к serial как 4.7×.
  • huge_few: throughput eager и serial практически одинаков.
  • huge_few: пиковый RSS eager относится к serial как 4.5×.
  • При balanced p95 time-to-ready для huge_few в 20.9× больше, чем для small_many: размер изображения меняет стоимость задания, а не только число заданий.
  • Compression vs scaling (balanced): exactly_right — encode share 83.1%, scale p95 1527 ms; huge_few — encode share 61.4%, scale p95 3448 ms. На exactly_right optimized WebP кодируется в native 1280, поэтому это нижняя граница стоимости сжатия; на huge_few к ней добавляется downscale.
  • На huge_few eager почти не поднял throughput (1.02×): worker ограничен 1.0 CPU, а декодирование и масштабирование упираются в CPU. Concurrency снижает queue lag, но при этом растёт RSS; это и есть компромисс latency/ресурсы, а не бесплатное ускорение.
  • serial ограничивает одновременно обрабатываемые задания одним, поэтому queue lag растёт, но RSS остаётся минимальным. eager держит больше заданий в работе и в prefetch-окне брокера.
  • Backpressure: cap=3 выставлен намеренно, поэтому число принятых заданий — следствие настройки, а не открытие. Значимо то, что API вернул 503 с Retry-After=5, а после старта worker очередь восстановилась за 7.7 с.
@@ -263,8 +339,15 @@ Max parallel — максимум одновременно обрабатыва
  • Один worker, один RabbitMQ и один PostgreSQL: это сравнение настроек, а не HA-кластер. Числа зависят от CPU/диска конкретной машины.
  • -
  • PNG-фикстуры синтетические и хорошо сжимаются; доминирует стоимость декодирования -и масштабирования, а не передача по сети. Размер в байтах указан в JSON.
  • +
  • PNG-фикстуры синтетические и хорошо сжимаются; absolute encode time зависит +от паттерна, но сравнение сценариев при одном паттерне остаётся корректным. +Размер в байтах указан в JSON.
  • +
  • Timing-поля читаются из worker-лога. Это wall-clock внутри процесса, поэтому +при concurrency > 1 в них входит и планировщик, что как раз и нужно для +сравнения профилей.
  • +
  • exactly_right кодирует 1.64 MP WebP, а huge_few после downscale — около +1.23 MP. Сценарий убирает scaling, но не уравнивает объём encode; это указано, +чтобы не сравнивать throughput как «одинаковую работу».
  • RSS сэмплируется из /proc/1/status; короткий прогон не заменяет длительный memory profile и p99.
  • Backpressure cap приблизительный: каждая API-реплика считает pending-задания diff --git a/evidence/track1.json b/evidence/track1.json index d9adae62..81fba6b0 100644 --- a/evidence/track1.json +++ b/evidence/track1.json @@ -1,8 +1,8 @@ { - "generated_at": "2026-11-14T21:42:03.122104+00:00", + "generated_at": "2026-11-20T16:02:07.856909+00:00", "environment": { "base_url": "http://172.18.0.1:8080", - "host": "48f08d2e9267", + "host": "8d2280b0dfc2", "platform": "Linux-6.12.73+deb13-amd64-x86_64-with-glibc2.39", "cpu_count": 4, "python": "3.12.3", @@ -15,7 +15,7 @@ "worker_mem_limit_default": "1G", "note": "Experimental workers use a larger memory limit than .env so the eager huge-image run can hold decoded images without OOM; CPU is still limited so throughput is not a host-CPU artifact." }, - "hypothesis": "Small images that fit the variants without large scaling should be cheaper per job; huge images force decode+downscale, so each job costs more memory and time. More concurrency should raise throughput and RSS but reduce queue lag.", + "hypothesis": "Job cost is decode + scale + encode. exactly_right (1280x1280) keeps the optimized WebP at native size, so it measures the compression floor; huge_few adds downscale, and small_many has few pixels. More concurrency should raise throughput and RSS but reduce queue lag.", "profiles": [ { "name": "serial", @@ -41,10 +41,18 @@ "name": "small_many", "width": 192, "height": 128, - "description": "many small images; variants fit or crop without large scaling", + "description": "many small images; almost no scaling", "count": 30, "bytes": 941 }, + { + "name": "exactly_right", + "width": 1280, + "height": 1280, + "description": "largest variant encodes at native 1280, so no scaling on the WebP path", + "count": 18, + "bytes": 23176 + }, { "name": "huge_few", "width": 2400, @@ -65,72 +73,94 @@ "image_height": 128, "image_bytes": 941, "image_sha256": "7e90807e4df32b24ab7e0f8f540c76b93ed2a0e9f91e7b3350748cfc8029a088", - "throughput_jobs_per_s": 6.875, - "pipeline_span_seconds": 4.364, - "submit_seconds": 0.987, - "wall_seconds": 4.922, - "upload_p50_ms": 43.4, - "upload_p95_ms": 99.8, - "queue_lag_p50_ms": 2030.8, - "queue_lag_p95_ms": 3193.7, - "queue_lag_max_ms": 3385.0, - "time_to_ready_p50_ms": 2119.7, - "time_to_ready_p95_ms": 3284.1, - "time_to_ready_max_ms": 3475.8, - "processing_p50_ms": 85.7, - "processing_p95_ms": 151.5, + "throughput_jobs_per_s": 6.837, + "pipeline_span_seconds": 4.388, + "submit_seconds": 1.051, + "wall_seconds": 4.99, + "upload_p50_ms": 67.7, + "upload_p95_ms": 148.2, + "queue_lag_p50_ms": 2029.3, + "queue_lag_p95_ms": 3256.2, + "queue_lag_max_ms": 3425.7, + "time_to_ready_p50_ms": 2112.6, + "time_to_ready_p95_ms": 3338.0, + "time_to_ready_max_ms": 3506.6, + "processing_p50_ms": 88.4, + "processing_p95_ms": 169.9, + "decode_ms_p50": 0.26, + "decode_ms_p95": 1.52, + "scale_ms_p50": 5.34, + "scale_ms_p95": 14.31, + "jpeg_encode_ms_p50": 2.37, + "jpeg_encode_ms_p95": 5.96, + "webp_encode_ms_p50": 28.57, + "webp_encode_ms_p95": 75.86, + "encode_ms_p50": 30.71, + "encode_ms_p95": 80.0, + "pipeline_ms_p50": 36.3, + "pipeline_ms_p95": 92.23, + "encode_share_pct": 83.8, + "timing_jobs": 30, + "original_bytes": 28230, + "variants_bytes": 469710, + "variant_bytes": { + "thumbnail": 214740, + "crop": 151290, + "optimized": 103680 + }, + "compression_ratio": 16.639, "max_backlog_jobs": 27, "max_parallel_jobs": 1, - "rss_baseline_mib": 20.3, + "rss_baseline_mib": 20.4, "rss_peak_mib": 28.2, - "rss_end_mib": 27.1, + "rss_end_mib": 27.6, "rss_samples": 63, - "rss_delta_mib": 7.9, + "rss_delta_mib": 7.8, "timeline": [ { - "t": 1.356, + "t": 1.408, "queued": 24, - "processing": 0, - "ready": 6, - "failed": 0 - }, - { - "t": 1.998, - "queued": 19, "processing": 1, - "ready": 10, + "ready": 5, "failed": 0 }, { - "t": 2.711, - "queued": 15, + "t": 2.169, + "queued": 19, "processing": 0, + "ready": 11, + "failed": 0 + }, + { + "t": 2.78, + "queued": 14, + "processing": 1, "ready": 15, "failed": 0 }, { - "t": 3.265, + "t": 3.314, "queued": 10, "processing": 1, "ready": 19, "failed": 0 }, { - "t": 3.754, + "t": 3.848, "queued": 5, - "processing": 1, - "ready": 24, + "processing": 0, + "ready": 25, "failed": 0 }, { - "t": 4.384, - "queued": 0, - "processing": 1, + "t": 4.465, + "queued": 1, + "processing": 0, "ready": 29, "failed": 0 }, { - "t": 4.922, + "t": 4.99, "queued": 0, "processing": 0, "ready": 30, @@ -138,6 +168,517 @@ } ] }, + { + "profile": "serial", + "scenario": "exactly_right", + "concurrency": 1, + "prefetch": 1, + "jobs": 18, + "image_width": 1280, + "image_height": 1280, + "image_bytes": 23176, + "image_sha256": "031edc2b1c3dc17d244613b1488661ffb44f5a70d425c0ec753287c3d5c41bdb", + "throughput_jobs_per_s": 0.475, + "pipeline_span_seconds": 37.868, + "submit_seconds": 0.7, + "wall_seconds": 38.475, + "upload_p50_ms": 63.0, + "upload_p95_ms": 130.9, + "queue_lag_p50_ms": 17769.1, + "queue_lag_p95_ms": 33590.9, + "queue_lag_max_ms": 35435.2, + "time_to_ready_p50_ms": 19857.3, + "time_to_ready_p95_ms": 35713.9, + "time_to_ready_max_ms": 37398.5, + "processing_p50_ms": 2034.6, + "processing_p95_ms": 2179.6, + "decode_ms_p50": 13.95, + "decode_ms_p95": 20.18, + "scale_ms_p50": 331.73, + "scale_ms_p95": 432.28, + "jpeg_encode_ms_p50": 4.58, + "jpeg_encode_ms_p95": 5.56, + "webp_encode_ms_p50": 1613.4, + "webp_encode_ms_p95": 1717.25, + "encode_ms_p50": 1617.88, + "encode_ms_p95": 1722.26, + "pipeline_ms_p50": 1976.81, + "pipeline_ms_p95": 2117.42, + "encode_share_pct": 82.0, + "timing_jobs": 18, + "original_bytes": 417168, + "variants_bytes": 4345668, + "variant_bytes": { + "thumbnail": 565254, + "crop": 142146, + "optimized": 3638268 + }, + "compression_ratio": 10.417, + "max_backlog_jobs": 18, + "max_parallel_jobs": 1, + "rss_baseline_mib": 20.5, + "rss_peak_mib": 174.3, + "rss_end_mib": 147.9, + "rss_samples": 382, + "rss_delta_mib": 153.8, + "timeline": [ + { + "t": 1.084, + "queued": 17, + "processing": 1, + "ready": 0, + "failed": 0 + }, + { + "t": 1.569, + "queued": 17, + "processing": 1, + "ready": 0, + "failed": 0 + }, + { + "t": 2.023, + "queued": 17, + "processing": 1, + "ready": 0, + "failed": 0 + }, + { + "t": 2.824, + "queued": 16, + "processing": 1, + "ready": 1, + "failed": 0 + }, + { + "t": 3.53, + "queued": 16, + "processing": 1, + "ready": 1, + "failed": 0 + }, + { + "t": 3.993, + "queued": 16, + "processing": 1, + "ready": 1, + "failed": 0 + }, + { + "t": 4.63, + "queued": 16, + "processing": 1, + "ready": 1, + "failed": 0 + }, + { + "t": 5.174, + "queued": 15, + "processing": 1, + "ready": 2, + "failed": 0 + }, + { + "t": 5.651, + "queued": 15, + "processing": 1, + "ready": 2, + "failed": 0 + }, + { + "t": 6.189, + "queued": 15, + "processing": 1, + "ready": 2, + "failed": 0 + }, + { + "t": 6.79, + "queued": 15, + "processing": 0, + "ready": 3, + "failed": 0 + }, + { + "t": 7.493, + "queued": 14, + "processing": 1, + "ready": 3, + "failed": 0 + }, + { + "t": 8.013, + "queued": 14, + "processing": 1, + "ready": 3, + "failed": 0 + }, + { + "t": 8.619, + "queued": 14, + "processing": 1, + "ready": 3, + "failed": 0 + }, + { + "t": 9.193, + "queued": 13, + "processing": 1, + "ready": 4, + "failed": 0 + }, + { + "t": 9.655, + "queued": 13, + "processing": 1, + "ready": 4, + "failed": 0 + }, + { + "t": 10.192, + "queued": 13, + "processing": 1, + "ready": 4, + "failed": 0 + }, + { + "t": 10.798, + "queued": 13, + "processing": 1, + "ready": 4, + "failed": 0 + }, + { + "t": 11.314, + "queued": 12, + "processing": 1, + "ready": 5, + "failed": 0 + }, + { + "t": 11.845, + "queued": 12, + "processing": 1, + "ready": 5, + "failed": 0 + }, + { + "t": 12.651, + "queued": 12, + "processing": 1, + "ready": 5, + "failed": 0 + }, + { + "t": 13.251, + "queued": 11, + "processing": 1, + "ready": 6, + "failed": 0 + }, + { + "t": 13.738, + "queued": 11, + "processing": 1, + "ready": 6, + "failed": 0 + }, + { + "t": 14.334, + "queued": 11, + "processing": 1, + "ready": 6, + "failed": 0 + }, + { + "t": 14.971, + "queued": 11, + "processing": 1, + "ready": 6, + "failed": 0 + }, + { + "t": 15.461, + "queued": 10, + "processing": 1, + "ready": 7, + "failed": 0 + }, + { + "t": 15.968, + "queued": 10, + "processing": 1, + "ready": 7, + "failed": 0 + }, + { + "t": 16.806, + "queued": 10, + "processing": 1, + "ready": 7, + "failed": 0 + }, + { + "t": 17.396, + "queued": 9, + "processing": 1, + "ready": 8, + "failed": 0 + }, + { + "t": 17.865, + "queued": 9, + "processing": 1, + "ready": 8, + "failed": 0 + }, + { + "t": 18.498, + "queued": 9, + "processing": 1, + "ready": 8, + "failed": 0 + }, + { + "t": 19.066, + "queued": 9, + "processing": 1, + "ready": 8, + "failed": 0 + }, + { + "t": 19.551, + "queued": 8, + "processing": 1, + "ready": 9, + "failed": 0 + }, + { + "t": 20.02, + "queued": 8, + "processing": 1, + "ready": 9, + "failed": 0 + }, + { + "t": 20.72, + "queued": 8, + "processing": 1, + "ready": 9, + "failed": 0 + }, + { + "t": 21.467, + "queued": 7, + "processing": 1, + "ready": 10, + "failed": 0 + }, + { + "t": 21.942, + "queued": 7, + "processing": 1, + "ready": 10, + "failed": 0 + }, + { + "t": 22.584, + "queued": 7, + "processing": 1, + "ready": 10, + "failed": 0 + }, + { + "t": 23.161, + "queued": 7, + "processing": 1, + "ready": 10, + "failed": 0 + }, + { + "t": 23.639, + "queued": 6, + "processing": 1, + "ready": 11, + "failed": 0 + }, + { + "t": 24.135, + "queued": 6, + "processing": 1, + "ready": 11, + "failed": 0 + }, + { + "t": 24.834, + "queued": 6, + "processing": 1, + "ready": 11, + "failed": 0 + }, + { + "t": 25.533, + "queued": 6, + "processing": 1, + "ready": 11, + "failed": 0 + }, + { + "t": 26.068, + "queued": 5, + "processing": 1, + "ready": 12, + "failed": 0 + }, + { + "t": 26.675, + "queued": 5, + "processing": 1, + "ready": 12, + "failed": 0 + }, + { + "t": 27.25, + "queued": 5, + "processing": 1, + "ready": 12, + "failed": 0 + }, + { + "t": 27.893, + "queued": 4, + "processing": 1, + "ready": 13, + "failed": 0 + }, + { + "t": 28.359, + "queued": 4, + "processing": 1, + "ready": 13, + "failed": 0 + }, + { + "t": 28.83, + "queued": 4, + "processing": 1, + "ready": 13, + "failed": 0 + }, + { + "t": 29.637, + "queued": 4, + "processing": 0, + "ready": 14, + "failed": 0 + }, + { + "t": 30.377, + "queued": 3, + "processing": 1, + "ready": 14, + "failed": 0 + }, + { + "t": 30.885, + "queued": 3, + "processing": 1, + "ready": 14, + "failed": 0 + }, + { + "t": 31.533, + "queued": 3, + "processing": 1, + "ready": 14, + "failed": 0 + }, + { + "t": 32.214, + "queued": 2, + "processing": 1, + "ready": 15, + "failed": 0 + }, + { + "t": 32.68, + "queued": 2, + "processing": 1, + "ready": 15, + "failed": 0 + }, + { + "t": 33.236, + "queued": 2, + "processing": 1, + "ready": 15, + "failed": 0 + }, + { + "t": 33.951, + "queued": 1, + "processing": 1, + "ready": 16, + "failed": 0 + }, + { + "t": 34.599, + "queued": 1, + "processing": 1, + "ready": 16, + "failed": 0 + }, + { + "t": 35.142, + "queued": 1, + "processing": 1, + "ready": 16, + "failed": 0 + }, + { + "t": 35.776, + "queued": 1, + "processing": 1, + "ready": 16, + "failed": 0 + }, + { + "t": 36.27, + "queued": 0, + "processing": 1, + "ready": 17, + "failed": 0 + }, + { + "t": 36.763, + "queued": 0, + "processing": 1, + "ready": 17, + "failed": 0 + }, + { + "t": 37.393, + "queued": 0, + "processing": 1, + "ready": 17, + "failed": 0 + }, + { + "t": 38.0, + "queued": 0, + "processing": 1, + "ready": 17, + "failed": 0 + }, + { + "t": 38.475, + "queued": 0, + "processing": 0, + "ready": 18, + "failed": 0 + } + ] + }, { "profile": "serial", "scenario": "huge_few", @@ -148,352 +689,367 @@ "image_height": 1800, "image_bytes": 53135, "image_sha256": "68c88fcbe5c4ba31ee745047e8be4dc64e5d7711f81f5e50a4276fb52cf925ba", - "throughput_jobs_per_s": 0.443, - "pipeline_span_seconds": 27.106, - "submit_seconds": 0.52, - "wall_seconds": 27.417, - "upload_p50_ms": 162.6, - "upload_p95_ms": 227.5, - "queue_lag_p50_ms": 12336.3, - "queue_lag_p95_ms": 23333.0, - "queue_lag_max_ms": 24511.6, - "time_to_ready_p50_ms": 14530.2, - "time_to_ready_p95_ms": 25521.0, - "time_to_ready_max_ms": 26776.2, - "processing_p50_ms": 2233.0, - "processing_p95_ms": 2311.1, + "throughput_jobs_per_s": 0.436, + "pipeline_span_seconds": 27.539, + "submit_seconds": 0.724, + "wall_seconds": 27.861, + "upload_p50_ms": 220.5, + "upload_p95_ms": 349.6, + "queue_lag_p50_ms": 12568.0, + "queue_lag_p95_ms": 23668.1, + "queue_lag_max_ms": 24910.1, + "time_to_ready_p50_ms": 14822.7, + "time_to_ready_p95_ms": 25896.4, + "time_to_ready_max_ms": 27051.7, + "processing_p50_ms": 2213.4, + "processing_p95_ms": 2359.1, + "decode_ms_p50": 36.44, + "decode_ms_p95": 43.65, + "scale_ms_p50": 751.12, + "scale_ms_p95": 836.9, + "jpeg_encode_ms_p50": 3.46, + "jpeg_encode_ms_p95": 4.62, + "webp_encode_ms_p50": 1366.74, + "webp_encode_ms_p95": 1504.85, + "encode_ms_p50": 1370.34, + "encode_ms_p95": 1508.26, + "pipeline_ms_p50": 2148.57, + "pipeline_ms_p95": 2287.69, + "encode_share_pct": 63.8, + "timing_jobs": 12, + "original_bytes": 637620, + "variants_bytes": 3163980, + "variant_bytes": { + "thumbnail": 260148, + "crop": 78672, + "optimized": 2825160 + }, + "compression_ratio": 4.962, "max_backlog_jobs": 12, "max_parallel_jobs": 1, - "rss_baseline_mib": 20.5, - "rss_peak_mib": 204.7, - "rss_end_mib": 203.1, - "rss_samples": 278, - "rss_delta_mib": 184.2, + "rss_baseline_mib": 20.4, + "rss_peak_mib": 212.9, + "rss_end_mib": 201.9, + "rss_samples": 282, + "rss_delta_mib": 192.5, "timeline": [ { - "t": 0.827, + "t": 1.103, "queued": 11, "processing": 1, "ready": 0, "failed": 0 }, { - "t": 1.494, + "t": 1.718, "queued": 11, "processing": 1, "ready": 0, "failed": 0 }, { - "t": 2.002, + "t": 2.373, "queued": 11, "processing": 1, "ready": 0, "failed": 0 }, { - "t": 2.488, + "t": 3.053, "queued": 10, "processing": 1, "ready": 1, "failed": 0 }, { - "t": 3.193, + "t": 3.561, "queued": 10, "processing": 1, "ready": 1, "failed": 0 }, { - "t": 4.011, + "t": 4.035, "queued": 10, "processing": 1, "ready": 1, "failed": 0 }, { - "t": 4.519, + "t": 4.705, "queued": 10, "processing": 1, "ready": 1, "failed": 0 }, { - "t": 5.153, + "t": 5.257, "queued": 9, "processing": 1, "ready": 2, "failed": 0 }, { - "t": 5.723, + "t": 5.731, "queued": 9, "processing": 1, "ready": 2, "failed": 0 }, { - "t": 6.187, + "t": 6.387, "queued": 9, "processing": 1, "ready": 2, "failed": 0 }, { - "t": 6.671, + "t": 7.217, "queued": 9, "processing": 1, "ready": 2, "failed": 0 }, { - "t": 7.283, + "t": 7.717, "queued": 8, "processing": 1, "ready": 3, "failed": 0 }, { - "t": 7.94, + "t": 8.301, "queued": 8, "processing": 1, "ready": 3, "failed": 0 }, { - "t": 8.452, + "t": 8.921, "queued": 8, "processing": 1, "ready": 3, "failed": 0 }, { - "t": 9.103, + "t": 9.555, "queued": 8, "processing": 1, "ready": 3, "failed": 0 }, { - "t": 9.694, + "t": 10.199, "queued": 7, "processing": 1, "ready": 4, "failed": 0 }, { - "t": 10.185, + "t": 10.721, "queued": 7, "processing": 1, "ready": 4, "failed": 0 }, { - "t": 10.674, + "t": 11.453, "queued": 7, "processing": 1, "ready": 4, "failed": 0 }, { - "t": 11.333, - "queued": 7, - "processing": 1, - "ready": 4, - "failed": 0 - }, - { - "t": 11.836, + "t": 12.053, "queued": 6, "processing": 1, "ready": 5, "failed": 0 }, { - "t": 12.386, + "t": 12.587, "queued": 6, "processing": 1, "ready": 5, "failed": 0 }, { - "t": 13.112, + "t": 13.065, "queued": 6, "processing": 1, "ready": 5, "failed": 0 }, { - "t": 13.692, + "t": 13.704, "queued": 6, "processing": 1, "ready": 5, "failed": 0 }, { - "t": 14.188, + "t": 14.234, "queued": 5, "processing": 1, "ready": 6, "failed": 0 }, { - "t": 14.684, + "t": 14.723, "queued": 5, "processing": 1, "ready": 6, "failed": 0 }, { - "t": 15.277, + "t": 15.371, "queued": 5, "processing": 1, "ready": 6, "failed": 0 }, { - "t": 15.82, + "t": 16.103, "queued": 5, "processing": 1, "ready": 6, "failed": 0 }, { - "t": 16.477, + "t": 16.663, "queued": 4, "processing": 1, "ready": 7, "failed": 0 }, { - "t": 17.103, + "t": 17.154, "queued": 4, "processing": 1, "ready": 7, "failed": 0 }, { - "t": 17.622, + "t": 17.803, "queued": 4, "processing": 1, "ready": 7, "failed": 0 }, { - "t": 18.203, + "t": 18.351, "queued": 4, "processing": 1, "ready": 7, "failed": 0 }, { - "t": 18.738, + "t": 18.842, "queued": 3, "processing": 1, "ready": 8, "failed": 0 }, { - "t": 19.242, + "t": 19.592, "queued": 3, "processing": 1, "ready": 8, "failed": 0 }, { - "t": 19.806, + "t": 20.327, "queued": 3, "processing": 1, "ready": 8, "failed": 0 }, { - "t": 20.406, + "t": 20.909, "queued": 3, "processing": 1, "ready": 8, "failed": 0 }, { - "t": 20.89, + "t": 21.545, "queued": 2, "processing": 1, "ready": 9, "failed": 0 }, { - "t": 21.44, + "t": 22.235, "queued": 2, "processing": 1, "ready": 9, "failed": 0 }, { - "t": 22.162, + "t": 22.7, "queued": 2, "processing": 1, "ready": 9, "failed": 0 }, { - "t": 22.773, + "t": 23.195, "queued": 2, "processing": 1, "ready": 9, "failed": 0 }, { - "t": 23.257, + "t": 23.816, "queued": 1, "processing": 1, "ready": 10, "failed": 0 }, { - "t": 23.854, + "t": 24.524, "queued": 1, "processing": 1, "ready": 10, "failed": 0 }, { - "t": 24.452, + "t": 25.041, "queued": 1, "processing": 1, "ready": 10, "failed": 0 }, { - "t": 24.927, - "queued": 1, - "processing": 1, - "ready": 10, - "failed": 0 - }, - { - "t": 25.472, + "t": 25.696, "queued": 0, "processing": 1, "ready": 11, "failed": 0 }, { - "t": 26.224, + "t": 26.264, "queued": 0, "processing": 1, "ready": 11, "failed": 0 }, { - "t": 26.95, + "t": 26.733, "queued": 0, "processing": 1, "ready": 11, "failed": 0 }, { - "t": 27.417, + "t": 27.275, + "queued": 0, + "processing": 1, + "ready": 11, + "failed": 0 + }, + { + "t": 27.861, "queued": 0, "processing": 0, "ready": 12, @@ -511,44 +1067,66 @@ "image_height": 128, "image_bytes": 941, "image_sha256": "7e90807e4df32b24ab7e0f8f540c76b93ed2a0e9f91e7b3350748cfc8029a088", - "throughput_jobs_per_s": 11.59, - "pipeline_span_seconds": 2.588, - "submit_seconds": 1.049, - "wall_seconds": 2.966, - "upload_p50_ms": 58.0, - "upload_p95_ms": 178.8, - "queue_lag_p50_ms": 1131.4, - "queue_lag_p95_ms": 1623.9, - "queue_lag_max_ms": 1647.8, - "time_to_ready_p50_ms": 1335.0, - "time_to_ready_p95_ms": 1735.8, - "time_to_ready_max_ms": 1756.1, - "processing_p50_ms": 211.1, - "processing_p95_ms": 336.3, - "max_backlog_jobs": 26, + "throughput_jobs_per_s": 13.694, + "pipeline_span_seconds": 2.191, + "submit_seconds": 1.072, + "wall_seconds": 2.852, + "upload_p50_ms": 57.2, + "upload_p95_ms": 157.6, + "queue_lag_p50_ms": 916.5, + "queue_lag_p95_ms": 1146.6, + "queue_lag_max_ms": 1174.2, + "time_to_ready_p50_ms": 1042.9, + "time_to_ready_p95_ms": 1273.7, + "time_to_ready_max_ms": 1324.2, + "processing_p50_ms": 149.2, + "processing_p95_ms": 396.0, + "decode_ms_p50": 0.27, + "decode_ms_p95": 1.17, + "scale_ms_p50": 6.66, + "scale_ms_p95": 17.26, + "jpeg_encode_ms_p50": 2.4, + "jpeg_encode_ms_p95": 7.88, + "webp_encode_ms_p50": 35.93, + "webp_encode_ms_p95": 87.15, + "encode_ms_p50": 38.7, + "encode_ms_p95": 89.96, + "pipeline_ms_p50": 49.27, + "pipeline_ms_p95": 109.71, + "encode_share_pct": 83.8, + "timing_jobs": 30, + "original_bytes": 28230, + "variants_bytes": 469710, + "variant_bytes": { + "thumbnail": 214740, + "crop": 151290, + "optimized": 103680 + }, + "compression_ratio": 16.639, + "max_backlog_jobs": 24, "max_parallel_jobs": 4, - "rss_baseline_mib": 20.5, - "rss_peak_mib": 30.6, - "rss_end_mib": 28.5, + "rss_baseline_mib": 20.4, + "rss_peak_mib": 28.9, + "rss_end_mib": 27.4, "rss_samples": 43, - "rss_delta_mib": 10.1, + "rss_delta_mib": 8.5, "timeline": [ { - "t": 1.679, - "queued": 14, - "processing": 4, + "t": 1.489, + "queued": 15, + "processing": 3, "ready": 12, "failed": 0 }, { - "t": 2.514, - "queued": 4, - "processing": 2, - "ready": 24, + "t": 2.222, + "queued": 0, + "processing": 3, + "ready": 27, "failed": 0 }, { - "t": 2.966, + "t": 2.852, "queued": 0, "processing": 0, "ready": 30, @@ -558,373 +1136,900 @@ }, { "profile": "balanced", - "scenario": "huge_few", + "scenario": "exactly_right", "concurrency": 4, "prefetch": 4, - "jobs": 12, - "image_width": 2400, - "image_height": 1800, - "image_bytes": 53135, - "image_sha256": "68c88fcbe5c4ba31ee745047e8be4dc64e5d7711f81f5e50a4276fb52cf925ba", - "throughput_jobs_per_s": 0.454, - "pipeline_span_seconds": 26.412, - "submit_seconds": 0.688, - "wall_seconds": 26.671, - "upload_p50_ms": 178.3, - "upload_p95_ms": 298.3, - "queue_lag_p50_ms": 8712.6, - "queue_lag_p95_ms": 17716.4, - "queue_lag_max_ms": 18009.6, - "time_to_ready_p50_ms": 17199.1, - "time_to_ready_p95_ms": 25784.6, - "time_to_ready_max_ms": 25919.7, - "processing_p50_ms": 8459.9, - "processing_p95_ms": 8709.7, - "max_backlog_jobs": 12, + "jobs": 18, + "image_width": 1280, + "image_height": 1280, + "image_bytes": 23176, + "image_sha256": "031edc2b1c3dc17d244613b1488661ffb44f5a70d425c0ec753287c3d5c41bdb", + "throughput_jobs_per_s": 0.495, + "pipeline_span_seconds": 36.391, + "submit_seconds": 0.745, + "wall_seconds": 36.739, + "upload_p50_ms": 68.9, + "upload_p95_ms": 129.2, + "queue_lag_p50_ms": 15501.0, + "queue_lag_p95_ms": 31510.3, + "queue_lag_max_ms": 31697.3, + "time_to_ready_p50_ms": 23200.0, + "time_to_ready_p95_ms": 35764.4, + "time_to_ready_max_ms": 35844.6, + "processing_p50_ms": 7711.4, + "processing_p95_ms": 8173.6, + "decode_ms_p50": 21.65, + "decode_ms_p95": 85.9, + "scale_ms_p50": 1134.16, + "scale_ms_p95": 1527.0, + "jpeg_encode_ms_p50": 6.13, + "jpeg_encode_ms_p95": 35.49, + "webp_encode_ms_p50": 6165.41, + "webp_encode_ms_p95": 6554.13, + "encode_ms_p50": 6174.86, + "encode_ms_p95": 6562.25, + "pipeline_ms_p50": 7376.17, + "pipeline_ms_p95": 8011.08, + "encode_share_pct": 83.1, + "timing_jobs": 18, + "original_bytes": 417168, + "variants_bytes": 4345668, + "variant_bytes": { + "thumbnail": 565254, + "crop": 142146, + "optimized": 3638268 + }, + "compression_ratio": 10.417, + "max_backlog_jobs": 18, "max_parallel_jobs": 4, "rss_baseline_mib": 20.4, - "rss_peak_mib": 728.7, - "rss_end_mib": 609.9, - "rss_samples": 254, - "rss_delta_mib": 708.3, + "rss_peak_mib": 433.8, + "rss_end_mib": 234.6, + "rss_samples": 346, + "rss_delta_mib": 413.4, "timeline": [ { - "t": 1.028, - "queued": 8, + "t": 1.145, + "queued": 14, "processing": 4, "ready": 0, "failed": 0 }, { - "t": 1.694, - "queued": 8, + "t": 1.641, + "queued": 14, "processing": 4, "ready": 0, "failed": 0 }, { - "t": 2.319, - "queued": 8, + "t": 2.489, + "queued": 14, "processing": 4, "ready": 0, "failed": 0 }, { - "t": 2.905, - "queued": 8, + "t": 3.204, + "queued": 14, "processing": 4, "ready": 0, "failed": 0 }, { - "t": 3.537, - "queued": 8, + "t": 3.668, + "queued": 14, "processing": 4, "ready": 0, "failed": 0 }, { - "t": 4.173, - "queued": 8, + "t": 4.342, + "queued": 14, "processing": 4, "ready": 0, "failed": 0 }, { - "t": 4.829, - "queued": 8, + "t": 4.881, + "queued": 14, "processing": 4, "ready": 0, "failed": 0 }, { - "t": 5.508, - "queued": 8, + "t": 5.367, + "queued": 14, "processing": 4, "ready": 0, "failed": 0 }, { - "t": 6.11, - "queued": 8, + "t": 5.967, + "queued": 14, "processing": 4, "ready": 0, "failed": 0 }, { - "t": 6.603, - "queued": 8, + "t": 6.779, + "queued": 14, "processing": 4, "ready": 0, "failed": 0 }, { - "t": 7.14, - "queued": 8, + "t": 7.316, + "queued": 14, "processing": 4, "ready": 0, "failed": 0 }, { - "t": 7.796, - "queued": 8, + "t": 7.926, + "queued": 14, "processing": 4, "ready": 0, "failed": 0 }, { - "t": 8.345, - "queued": 8, + "t": 8.666, + "queued": 11, "processing": 4, - "ready": 0, + "ready": 3, "failed": 0 }, { - "t": 8.854, - "queued": 8, - "processing": 4, - "ready": 0, - "failed": 0 - }, - { - "t": 9.667, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 10.488, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 11.103, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 11.769, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 12.384, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 12.857, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 13.371, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 14.228, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 14.929, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 15.574, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 16.234, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 16.711, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 17.257, - "queued": 4, - "processing": 4, - "ready": 4, - "failed": 0 - }, - { - "t": 17.945, - "queued": 2, + "t": 9.145, + "queued": 11, "processing": 3, - "ready": 7, + "ready": 4, "failed": 0 }, { - "t": 18.674, - "queued": 1, + "t": 9.675, + "queued": 10, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 10.329, + "queued": 10, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 10.924, + "queued": 10, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 11.419, + "queued": 10, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 12.149, + "queued": 10, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 12.764, + "queued": 10, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 13.242, + "queued": 10, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 13.715, + "queued": 10, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 14.363, + "queued": 10, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 14.939, + "queued": 10, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 15.461, + "queued": 10, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 16.279, + "queued": 8, + "processing": 4, + "ready": 6, + "failed": 0 + }, + { + "t": 17.024, + "queued": 7, "processing": 3, "ready": 8, "failed": 0 }, { - "t": 19.251, - "queued": 0, + "t": 17.515, + "queued": 6, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 19.997, - "queued": 0, + "t": 18.146, + "queued": 6, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 20.662, - "queued": 0, + "t": 18.745, + "queued": 6, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 21.146, - "queued": 0, + "t": 19.224, + "queued": 6, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 21.758, - "queued": 0, + "t": 19.778, + "queued": 6, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 22.395, - "queued": 0, + "t": 20.561, + "queued": 6, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 22.945, - "queued": 0, + "t": 21.335, + "queued": 6, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 23.693, - "queued": 0, + "t": 22.006, + "queued": 6, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 24.381, - "queued": 0, + "t": 22.619, + "queued": 6, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 24.906, - "queued": 0, + "t": 23.122, + "queued": 6, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 25.597, - "queued": 0, + "t": 23.625, + "queued": 6, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 26.174, - "queued": 0, - "processing": 4, - "ready": 8, + "t": 24.239, + "queued": 5, + "processing": 3, + "ready": 10, "failed": 0 }, { - "t": 26.671, - "queued": 0, - "processing": 0, + "t": 25.008, + "queued": 4, + "processing": 2, "ready": 12, "failed": 0 - } - ] - }, - { - "profile": "eager", - "scenario": "small_many", - "concurrency": 8, - "prefetch": 32, - "jobs": 30, - "image_width": 192, - "image_height": 128, - "image_bytes": 941, - "image_sha256": "7e90807e4df32b24ab7e0f8f540c76b93ed2a0e9f91e7b3350748cfc8029a088", - "throughput_jobs_per_s": 13.858, - "pipeline_span_seconds": 2.165, - "submit_seconds": 1.168, - "wall_seconds": 2.318, - "upload_p50_ms": 46.4, - "upload_p95_ms": 126.7, - "queue_lag_p50_ms": 749.2, - "queue_lag_p95_ms": 897.8, - "queue_lag_max_ms": 1052.1, - "time_to_ready_p50_ms": 1013.5, - "time_to_ready_p95_ms": 1157.8, - "time_to_ready_max_ms": 1175.0, - "processing_p50_ms": 271.8, - "processing_p95_ms": 565.9, - "max_backlog_jobs": 26, - "max_parallel_jobs": 8, - "rss_baseline_mib": 20.4, - "rss_peak_mib": 32.7, - "rss_end_mib": 28.8, - "rss_samples": 37, - "rss_delta_mib": 12.3, - "timeline": [ + }, { - "t": 1.637, - "queued": 10, - "processing": 6, + "t": 25.639, + "queued": 2, + "processing": 4, + "ready": 12, + "failed": 0 + }, + { + "t": 26.343, + "queued": 2, + "processing": 4, + "ready": 12, + "failed": 0 + }, + { + "t": 26.887, + "queued": 2, + "processing": 4, + "ready": 12, + "failed": 0 + }, + { + "t": 27.361, + "queued": 2, + "processing": 4, + "ready": 12, + "failed": 0 + }, + { + "t": 27.967, + "queued": 2, + "processing": 4, + "ready": 12, + "failed": 0 + }, + { + "t": 28.567, + "queued": 2, + "processing": 4, + "ready": 12, + "failed": 0 + }, + { + "t": 29.116, + "queued": 2, + "processing": 4, + "ready": 12, + "failed": 0 + }, + { + "t": 29.723, + "queued": 2, + "processing": 4, + "ready": 12, + "failed": 0 + }, + { + "t": 30.548, + "queued": 2, + "processing": 4, + "ready": 12, + "failed": 0 + }, + { + "t": 31.201, + "queued": 2, + "processing": 4, + "ready": 12, + "failed": 0 + }, + { + "t": 31.755, + "queued": 2, + "processing": 4, + "ready": 12, + "failed": 0 + }, + { + "t": 32.394, + "queued": 1, + "processing": 3, "ready": 14, "failed": 0 }, { - "t": 2.318, + "t": 32.993, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 33.567, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 34.321, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 35.03, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 35.509, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 36.148, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 36.739, + "queued": 0, + "processing": 0, + "ready": 18, + "failed": 0 + } + ] + }, + { + "profile": "balanced", + "scenario": "huge_few", + "concurrency": 4, + "prefetch": 4, + "jobs": 12, + "image_width": 2400, + "image_height": 1800, + "image_bytes": 53135, + "image_sha256": "68c88fcbe5c4ba31ee745047e8be4dc64e5d7711f81f5e50a4276fb52cf925ba", + "throughput_jobs_per_s": 0.441, + "pipeline_span_seconds": 27.197, + "submit_seconds": 0.699, + "wall_seconds": 27.842, + "upload_p50_ms": 234.9, + "upload_p95_ms": 340.5, + "queue_lag_p50_ms": 9161.6, + "queue_lag_p95_ms": 18160.7, + "queue_lag_max_ms": 18615.8, + "time_to_ready_p50_ms": 17676.9, + "time_to_ready_p95_ms": 26575.8, + "time_to_ready_max_ms": 26687.1, + "processing_p50_ms": 8751.2, + "processing_p95_ms": 9046.1, + "decode_ms_p50": 75.2, + "decode_ms_p95": 173.97, + "scale_ms_p50": 3244.52, + "scale_ms_p95": 3447.76, + "jpeg_encode_ms_p50": 4.08, + "jpeg_encode_ms_p95": 72.28, + "webp_encode_ms_p50": 5224.87, + "webp_encode_ms_p95": 5601.71, + "encode_ms_p50": 5254.45, + "encode_ms_p95": 5605.8, + "pipeline_ms_p50": 8502.19, + "pipeline_ms_p95": 8893.1, + "encode_share_pct": 61.4, + "timing_jobs": 12, + "original_bytes": 637620, + "variants_bytes": 3163980, + "variant_bytes": { + "thumbnail": 260148, + "crop": 78672, + "optimized": 2825160 + }, + "compression_ratio": 4.962, + "max_backlog_jobs": 12, + "max_parallel_jobs": 4, + "rss_baseline_mib": 18.4, + "rss_peak_mib": 756.4, + "rss_end_mib": 700.1, + "rss_samples": 264, + "rss_delta_mib": 738.0, + "timeline": [ + { + "t": 1.054, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 1.69, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 2.18, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 2.852, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 3.554, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 4.119, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 4.796, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 5.369, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 5.899, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 6.538, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 7.201, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 7.742, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 8.363, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 9.121, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 9.736, + "queued": 5, + "processing": 3, + "ready": 4, + "failed": 0 + }, + { + "t": 10.236, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 10.887, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 11.445, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 11.968, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 12.787, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 13.713, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 14.188, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 14.848, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 15.451, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 16.048, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 16.526, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 17.073, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 17.941, + "queued": 4, + "processing": 3, + "ready": 5, + "failed": 0 + }, + { + "t": 18.562, + "queued": 1, + "processing": 4, + "ready": 7, + "failed": 0 + }, + { + "t": 19.062, + "queued": 1, + "processing": 3, + "ready": 8, + "failed": 0 + }, + { + "t": 19.715, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 20.364, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 20.897, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 21.7, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 22.547, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 23.064, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 23.704, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 24.28, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 24.785, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 25.397, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 26.09, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 26.685, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 27.269, + "queued": 0, + "processing": 1, + "ready": 11, + "failed": 0 + }, + { + "t": 27.842, + "queued": 0, + "processing": 0, + "ready": 12, + "failed": 0 + } + ] + }, + { + "profile": "eager", + "scenario": "small_many", + "concurrency": 8, + "prefetch": 32, + "jobs": 30, + "image_width": 192, + "image_height": 128, + "image_bytes": 941, + "image_sha256": "7e90807e4df32b24ab7e0f8f540c76b93ed2a0e9f91e7b3350748cfc8029a088", + "throughput_jobs_per_s": 15.576, + "pipeline_span_seconds": 1.926, + "submit_seconds": 0.991, + "wall_seconds": 2.218, + "upload_p50_ms": 49.4, + "upload_p95_ms": 140.4, + "queue_lag_p50_ms": 620.0, + "queue_lag_p95_ms": 923.7, + "queue_lag_max_ms": 993.7, + "time_to_ready_p50_ms": 876.8, + "time_to_ready_p95_ms": 1165.8, + "time_to_ready_max_ms": 1192.1, + "processing_p50_ms": 288.9, + "processing_p95_ms": 410.2, + "decode_ms_p50": 0.28, + "decode_ms_p95": 2.92, + "scale_ms_p50": 11.84, + "scale_ms_p95": 39.22, + "jpeg_encode_ms_p50": 3.47, + "jpeg_encode_ms_p95": 25.67, + "webp_encode_ms_p50": 53.31, + "webp_encode_ms_p95": 88.79, + "encode_ms_p50": 60.16, + "encode_ms_p95": 94.53, + "pipeline_ms_p50": 70.72, + "pipeline_ms_p95": 116.55, + "encode_share_pct": 79.9, + "timing_jobs": 30, + "original_bytes": 28230, + "variants_bytes": 469710, + "variant_bytes": { + "thumbnail": 214740, + "crop": 151290, + "optimized": 103680 + }, + "compression_ratio": 16.639, + "max_backlog_jobs": 22, + "max_parallel_jobs": 8, + "rss_baseline_mib": 20.4, + "rss_peak_mib": 32.1, + "rss_end_mib": 31.3, + "rss_samples": 36, + "rss_delta_mib": 11.7, + "timeline": [ + { + "t": 1.583, + "queued": 6, + "processing": 5, + "ready": 19, + "failed": 0 + }, + { + "t": 2.218, "queued": 0, "processing": 0, "ready": 30, @@ -932,6 +2037,482 @@ } ] }, + { + "profile": "eager", + "scenario": "exactly_right", + "concurrency": 8, + "prefetch": 32, + "jobs": 18, + "image_width": 1280, + "image_height": 1280, + "image_bytes": 23176, + "image_sha256": "031edc2b1c3dc17d244613b1488661ffb44f5a70d425c0ec753287c3d5c41bdb", + "throughput_jobs_per_s": 0.495, + "pipeline_span_seconds": 36.374, + "submit_seconds": 0.732, + "wall_seconds": 37.056, + "upload_p50_ms": 99.8, + "upload_p95_ms": 143.3, + "queue_lag_p50_ms": 16204.1, + "queue_lag_p95_ms": 31734.6, + "queue_lag_max_ms": 31738.1, + "time_to_ready_p50_ms": 31582.0, + "time_to_ready_p95_ms": 35797.0, + "time_to_ready_max_ms": 35874.2, + "processing_p50_ms": 15651.6, + "processing_p95_ms": 16362.7, + "decode_ms_p50": 15.19, + "decode_ms_p95": 79.65, + "scale_ms_p50": 2513.98, + "scale_ms_p95": 3215.38, + "jpeg_encode_ms_p50": 5.52, + "jpeg_encode_ms_p95": 110.2, + "webp_encode_ms_p50": 12503.33, + "webp_encode_ms_p95": 13300.65, + "encode_ms_p50": 12510.13, + "encode_ms_p95": 13330.36, + "pipeline_ms_p50": 15115.26, + "pipeline_ms_p95": 15839.92, + "encode_share_pct": 83.1, + "timing_jobs": 18, + "original_bytes": 417168, + "variants_bytes": 4345668, + "variant_bytes": { + "thumbnail": 565254, + "crop": 142146, + "optimized": 3638268 + }, + "compression_ratio": 10.417, + "max_backlog_jobs": 18, + "max_parallel_jobs": 8, + "rss_baseline_mib": 20.5, + "rss_peak_mib": 818.4, + "rss_end_mib": 267.5, + "rss_samples": 348, + "rss_delta_mib": 797.9, + "timeline": [ + { + "t": 1.029, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 1.517, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 2.383, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 3.225, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 3.785, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 4.401, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 4.895, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 5.386, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 6.031, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 6.66, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 7.187, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 7.911, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 8.609, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 9.088, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 9.591, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 10.224, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 10.79, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 11.355, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 12.104, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 12.728, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 13.258, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 13.882, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 14.512, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 14.99, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 15.545, + "queued": 10, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 16.347, + "queued": 10, + "processing": 7, + "ready": 1, + "failed": 0 + }, + { + "t": 17.27, + "queued": 5, + "processing": 5, + "ready": 8, + "failed": 0 + }, + { + "t": 17.918, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 18.571, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 19.099, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 19.6, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 20.256, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 20.912, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 21.467, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 22.107, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 22.686, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 23.202, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 23.706, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 24.31, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 25.015, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 25.809, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 26.38, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 27.044, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 27.652, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 28.125, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 28.638, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 29.32, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 29.998, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 30.527, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 31.164, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 31.762, + "queued": 2, + "processing": 8, + "ready": 8, + "failed": 0 + }, + { + "t": 32.258, + "queued": 2, + "processing": 6, + "ready": 10, + "failed": 0 + }, + { + "t": 32.918, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 33.555, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 34.143, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 34.717, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 35.406, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 35.974, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 36.491, + "queued": 0, + "processing": 2, + "ready": 16, + "failed": 0 + }, + { + "t": 37.056, + "queued": 0, + "processing": 0, + "ready": 18, + "failed": 0 + } + ] + }, { "profile": "eager", "scenario": "huge_few", @@ -942,317 +2523,353 @@ "image_height": 1800, "image_bytes": 53135, "image_sha256": "68c88fcbe5c4ba31ee745047e8be4dc64e5d7711f81f5e50a4276fb52cf925ba", - "throughput_jobs_per_s": 0.459, - "pipeline_span_seconds": 26.164, - "submit_seconds": 0.621, - "wall_seconds": 26.514, - "upload_p50_ms": 211.1, - "upload_p95_ms": 334.8, - "queue_lag_p50_ms": 603.5, - "queue_lag_p95_ms": 17537.9, - "queue_lag_max_ms": 17742.4, - "time_to_ready_p50_ms": 17932.3, - "time_to_ready_p95_ms": 25698.9, - "time_to_ready_max_ms": 25761.3, - "processing_p50_ms": 16723.3, - "processing_p95_ms": 17319.7, + "throughput_jobs_per_s": 0.443, + "pipeline_span_seconds": 27.077, + "submit_seconds": 0.615, + "wall_seconds": 27.714, + "upload_p50_ms": 242.8, + "upload_p95_ms": 345.4, + "queue_lag_p50_ms": 464.0, + "queue_lag_p95_ms": 17648.0, + "queue_lag_max_ms": 17738.0, + "time_to_ready_p50_ms": 17949.7, + "time_to_ready_p95_ms": 26641.1, + "time_to_ready_max_ms": 26645.8, + "processing_p50_ms": 17312.3, + "processing_p95_ms": 17677.9, + "decode_ms_p50": 156.61, + "decode_ms_p95": 291.96, + "scale_ms_p50": 5794.3, + "scale_ms_p95": 6868.84, + "jpeg_encode_ms_p50": 60.28, + "jpeg_encode_ms_p95": 139.01, + "webp_encode_ms_p50": 9796.74, + "webp_encode_ms_p95": 11033.62, + "encode_ms_p50": 9844.68, + "encode_ms_p95": 11124.45, + "pipeline_ms_p50": 16593.78, + "pipeline_ms_p95": 17069.69, + "encode_share_pct": 61.4, + "timing_jobs": 12, + "original_bytes": 637620, + "variants_bytes": 3163980, + "variant_bytes": { + "thumbnail": 260148, + "crop": 78672, + "optimized": 2825160 + }, + "compression_ratio": 4.962, "max_backlog_jobs": 12, "max_parallel_jobs": 8, - "rss_baseline_mib": 20.0, - "rss_peak_mib": 1122.1, - "rss_end_mib": 763.8, - "rss_samples": 254, - "rss_delta_mib": 1102.1, + "rss_baseline_mib": 18.5, + "rss_peak_mib": 949.6, + "rss_end_mib": 818.5, + "rss_samples": 264, + "rss_delta_mib": 931.1, "timeline": [ { - "t": 1.084, - "queued": 6, - "processing": 6, + "t": 1.018, + "queued": 7, + "processing": 5, "ready": 0, "failed": 0 }, { - "t": 1.861, + "t": 1.526, "queued": 5, "processing": 7, "ready": 0, "failed": 0 }, { - "t": 2.395, + "t": 2.367, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 3.195, + "t": 3.265, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 3.804, + "t": 3.777, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 4.307, + "t": 4.415, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 4.914, + "t": 4.983, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 5.49, + "t": 5.475, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 5.99, + "t": 6.15, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 6.547, + "t": 6.969, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 7.386, + "t": 7.633, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 8.066, + "t": 8.371, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 8.621, + "t": 8.987, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 9.226, + "t": 9.457, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 9.729, + "t": 10.148, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 10.222, + "t": 10.767, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 10.837, + "t": 11.329, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 11.613, + "t": 11.952, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 12.158, + "t": 12.675, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 12.822, + "t": 13.232, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 13.409, + "t": 13.711, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 13.899, + "t": 14.346, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 14.425, + "t": 14.976, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 15.051, + "t": 15.633, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 15.644, + "t": 16.204, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 16.265, + "t": 16.721, "queued": 4, "processing": 8, "ready": 0, "failed": 0 }, { - "t": 17.048, + "t": 17.341, "queued": 4, - "processing": 7, - "ready": 1, + "processing": 8, + "ready": 0, "failed": 0 }, { - "t": 17.714, + "t": 17.954, "queued": 3, - "processing": 5, - "ready": 4, + "processing": 7, + "ready": 2, "failed": 0 }, { - "t": 18.216, - "queued": 1, + "t": 18.478, + "queued": 0, "processing": 6, - "ready": 5, + "ready": 6, "failed": 0 }, { - "t": 18.891, + "t": 19.183, + "queued": 0, + "processing": 5, + "ready": 7, + "failed": 0 + }, + { + "t": 19.779, "queued": 0, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 19.529, + "t": 20.313, "queued": 0, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 20.017, + "t": 20.928, "queued": 0, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 20.748, + "t": 21.683, "queued": 0, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 21.529, + "t": 22.289, "queued": 0, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 22.067, + "t": 22.799, "queued": 0, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 22.652, + "t": 23.462, "queued": 0, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 23.251, + "t": 23.964, "queued": 0, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 23.924, + "t": 24.448, "queued": 0, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 24.548, + "t": 25.17, "queued": 0, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 25.071, + "t": 25.999, "queued": 0, "processing": 4, "ready": 8, "failed": 0 }, { - "t": 25.888, + "t": 26.515, "queued": 0, - "processing": 3, - "ready": 9, + "processing": 4, + "ready": 8, "failed": 0 }, { - "t": 26.514, + "t": 27.171, + "queued": 0, + "processing": 2, + "ready": 10, + "failed": 0 + }, + { + "t": 27.714, "queued": 0, "processing": 0, "ready": 12, @@ -1267,28 +2884,28 @@ { "index": 0, "status": 202, - "elapsed_ms": 18.2, + "elapsed_ms": 26.7, "retry_after": null, "error_code": null }, { "index": 1, "status": 202, - "elapsed_ms": 17.4, + "elapsed_ms": 20.3, "retry_after": null, "error_code": null }, { "index": 2, "status": 202, - "elapsed_ms": 10.8, + "elapsed_ms": 13.8, "retry_after": null, "error_code": null }, { "index": 3, "status": 503, - "elapsed_ms": 3.9, + "elapsed_ms": 2.4, "retry_after": "5", "error_code": "unavailable" } @@ -1296,7 +2913,7 @@ "accepted_before_503": 3, "blocked_status": 503, "blocked_retry_after": "5", - "recovery_seconds": 7.596, + "recovery_seconds": 7.653, "recovered_jobs": 3, "probe_status": 202, "probe_ready": true, diff --git a/evidence/track1.md b/evidence/track1.md index 856c38ce..f2a0afad 100644 --- a/evidence/track1.md +++ b/evidence/track1.md @@ -1,16 +1,19 @@ # Lab5 track 1 — queue under load -Generated: `2026-11-14T21:42:03.122104+00:00` +Generated: `2026-11-20T16:02:07.856909+00:00` Synthetic images; no tokens or credentials are stored. -| Profile | Scenario | Jobs | Jobs/s | Queue lag p95, ms | Ready p95, ms | RSS peak, MiB | -| --- | --- | ---: | ---: | ---: | ---: | ---: | -| serial | small_many | 30 | 6.88 | 3194 | 3284 | 28 | -| serial | huge_few | 12 | 0.44 | 23333 | 25521 | 205 | -| balanced | small_many | 30 | 11.59 | 1624 | 1736 | 31 | -| balanced | huge_few | 12 | 0.45 | 17716 | 25785 | 729 | -| eager | small_many | 30 | 13.86 | 898 | 1158 | 33 | -| eager | huge_few | 12 | 0.46 | 17538 | 25699 | 1122 | +| Profile | Scenario | Jobs | Jobs/s | Queue lag p95, ms | Ready p95, ms | Encode p95, ms | Encode share, % | RSS peak, MiB | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| serial | small_many | 30 | 6.84 | 3256 | 3338 | 80 | 83.8 | 28 | +| serial | exactly_right | 18 | 0.47 | 33591 | 35714 | 1722 | 82.0 | 174 | +| serial | huge_few | 12 | 0.44 | 23668 | 25896 | 1508 | 63.8 | 213 | +| balanced | small_many | 30 | 13.69 | 1147 | 1274 | 90 | 83.8 | 29 | +| balanced | exactly_right | 18 | 0.49 | 31510 | 35764 | 6562 | 83.1 | 434 | +| balanced | huge_few | 12 | 0.44 | 18161 | 26576 | 5606 | 61.4 | 756 | +| eager | small_many | 30 | 15.58 | 924 | 1166 | 95 | 79.9 | 32 | +| eager | exactly_right | 18 | 0.49 | 31735 | 35797 | 13330 | 83.1 | 818 | +| eager | huge_few | 12 | 0.44 | 17648 | 26641 | 11124 | 61.4 | 950 | -Backpressure: cap=3, accepted=3, first 503 Retry-After=5, recovery=7.60s, probe HTTP 202. +Backpressure: cap=3 (deliberate), first 503 Retry-After=5, recovery=7.65s, probe HTTP 202. diff --git a/scripts/track1_experiment.py b/scripts/track1_experiment.py index 56731244..5c5566ab 100755 --- a/scripts/track1_experiment.py +++ b/scripts/track1_experiment.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -"""Lab5 grade-5 track 1: queue behaviour under small vs huge images. +"""Track 1: queue behaviour under small, exact and huge images. The experiment is deliberately short and self-contained: profiles: serial (concurrency 1, prefetch 1) balanced (4/4) eager (8/32) - scenarios: small_many (30 x 192x128 PNG, variants mostly fit without scaling) - huge_few (12 x 2400x1800 PNG, every job needs heavy scaling) + scenarios: small_many (30 x 192x128 PNG, almost no scaling) + exactly_right (18 x 1280x1280 PNG, optimized WebP at native size) + huge_few (12 x 2400x1800 PNG, every path needs heavy scaling) For each cell the worker is recreated with the profile, the images are uploaded concurrently, and the script records: @@ -16,6 +17,8 @@ concurrently, and the script records: * queue lag: created_at -> processing_started_at (95th percentile), * time-to-ready: created_at -> finished_at (p50/p95), * worker RSS baseline/peak sampled from /proc/1/status, + * decode / scale / JPEG / WebP timing from the worker's ready log, + * variant bytes versus the original (compression ratio), * upload latency and max in-flight jobs. A final backpressure run caps MAX_PENDING_JOBS at 3, stops the worker, shows @@ -40,6 +43,7 @@ import json import math import os import platform +import re import struct import subprocess import sys @@ -79,7 +83,9 @@ PROFILES = [ SCENARIOS = [ {"name": "small_many", "width": 192, "height": 128, - "description": "many small images; variants fit or crop without large scaling"}, + "description": "many small images; almost no scaling"}, + {"name": "exactly_right", "width": 1280, "height": 1280, + "description": "largest variant encodes at native 1280, so no scaling on the WebP path"}, {"name": "huge_few", "width": 2400, "height": 1800, "description": "few 4.32 MP images; optimized variant must scale down to 1280"}, ] @@ -330,6 +336,68 @@ def wait_for_terminal(attachment_ids, started_at, timeout=240): raise Failure(f"jobs did not finish within {timeout}s; last={rows}") +LOGFIELD_RE = re.compile(r'(\w+)=("(?:[^"\\]|\\.)*"|\S+)') + + +def job_timings(attachment_ids): + """Read the worker's ready log. The worker is recreated per run, so the + lines belong to this cell; the attachment ID is still checked explicitly.""" + if not attachment_ids: + return {} + wanted = set(attachment_ids) + output = compose_capture("logs", "--no-color", "--no-log-prefix", "worker") + timings = {} + for line in output.splitlines(): + if 'msg="attachment ready"' not in line: + continue + fields = {key: value.strip('"') for key, value in LOGFIELD_RE.findall(line)} + attachment_id = fields.get("attachment_id") + if attachment_id not in wanted: + continue + try: + timings[attachment_id] = { + name: int(fields[name]) + for name in ( + "decode_us", "scale_us", "jpeg_encode_us", + "webp_encode_us", "pipeline_us", + ) + } + except (KeyError, ValueError): + continue + return timings + + +def variant_stats(attachment_ids): + """Variant sizes come from the committed metadata, not from the client.""" + if not attachment_ids: + return {} + quoted = ",".join("'" + item + "'" for item in attachment_ids) + sql = ( + "SELECT a.id, a.size_bytes, " + "coalesce(sum((v->>'size_bytes')::bigint), 0), " + "coalesce(sum((v->>'size_bytes')::bigint) FILTER (WHERE v->>'name' = 'thumbnail'), 0), " + "coalesce(sum((v->>'size_bytes')::bigint) FILTER (WHERE v->>'name' = 'crop'), 0), " + "coalesce(sum((v->>'size_bytes')::bigint) FILTER (WHERE v->>'name' = 'optimized'), 0) " + "FROM attachments a " + "LEFT JOIN LATERAL jsonb_array_elements(a.variants) v ON true " + f"WHERE a.id IN ({quoted}) " + "GROUP BY a.id, a.size_bytes" + ) + result = {} + for line in psql(sql): + parts = line.split("|") + if len(parts) != 6: + continue + result[parts[0]] = { + "original_bytes": int(parts[1]), + "variants_bytes": int(parts[2]), + "thumbnail_bytes": int(parts[3]), + "crop_bytes": int(parts[4]), + "optimized_bytes": int(parts[5]), + } + return result + + def measure_run(base_url, actor, chat_id, profile, scenario, count, data, digest, cpu_limit="1.0", mem_limit="2G"): wait_drained() @@ -371,6 +439,45 @@ def measure_run(base_url, actor, chat_id, profile, scenario, count, data, digest if failed: raise Failure(f"jobs failed in {profile['name']}/{scenario['name']}: {failed[:2]}") + row_ids = [row["attachment_id"] for row in rows] + timings = job_timings(row_ids) + if len(timings) != len(rows): + raise Failure( + f"timing logs found for {len(timings)}/{len(rows)} jobs in " + f"{profile['name']}/{scenario['name']}" + ) + variants = variant_stats(row_ids) + if len(variants) != len(rows): + raise Failure( + f"variant metadata found for {len(variants)}/{len(rows)} jobs in " + f"{profile['name']}/{scenario['name']}" + ) + + decode_ms = [timings[item]["decode_us"] / 1000 for item in row_ids] + scale_ms = [timings[item]["scale_us"] / 1000 for item in row_ids] + jpeg_ms = [timings[item]["jpeg_encode_us"] / 1000 for item in row_ids] + webp_ms = [timings[item]["webp_encode_us"] / 1000 for item in row_ids] + encode_ms = [jpeg_ms[index] + webp_ms[index] for index in range(len(row_ids))] + pipeline_ms = [timings[item]["pipeline_us"] / 1000 for item in row_ids] + cpu_us = sum( + timings[item]["decode_us"] + timings[item]["scale_us"] + + timings[item]["jpeg_encode_us"] + timings[item]["webp_encode_us"] + for item in row_ids + ) + encode_us = sum( + timings[item]["jpeg_encode_us"] + timings[item]["webp_encode_us"] + for item in row_ids + ) + encode_share_pct = 100 * encode_us / cpu_us if cpu_us else None + + variant_bytes = { + name: sum(item[f"{name}_bytes"] for item in variants.values()) + for name in ("thumbnail", "crop", "optimized") + } + original_bytes = sum(item["original_bytes"] for item in variants.values()) + variants_bytes = sum(item["variants_bytes"] for item in variants.values()) + compression_ratio = variants_bytes / original_bytes if original_bytes else None + queue_lag = [ (row["processing_started_at"] - row["created_at"]) * 1000 for row in rows @@ -446,6 +553,24 @@ def measure_run(base_url, actor, chat_id, profile, scenario, count, data, digest "time_to_ready_max_ms": r(max(time_to_ready) if time_to_ready else None, 1), "processing_p50_ms": r(percentile(processing, 50), 1), "processing_p95_ms": r(percentile(processing, 95), 1), + "decode_ms_p50": r(percentile(decode_ms, 50), 2), + "decode_ms_p95": r(percentile(decode_ms, 95), 2), + "scale_ms_p50": r(percentile(scale_ms, 50), 2), + "scale_ms_p95": r(percentile(scale_ms, 95), 2), + "jpeg_encode_ms_p50": r(percentile(jpeg_ms, 50), 2), + "jpeg_encode_ms_p95": r(percentile(jpeg_ms, 95), 2), + "webp_encode_ms_p50": r(percentile(webp_ms, 50), 2), + "webp_encode_ms_p95": r(percentile(webp_ms, 95), 2), + "encode_ms_p50": r(percentile(encode_ms, 50), 2), + "encode_ms_p95": r(percentile(encode_ms, 95), 2), + "pipeline_ms_p50": r(percentile(pipeline_ms, 50), 2), + "pipeline_ms_p95": r(percentile(pipeline_ms, 95), 2), + "encode_share_pct": r(encode_share_pct, 1), + "timing_jobs": len(timings), + "original_bytes": original_bytes, + "variants_bytes": variants_bytes, + "variant_bytes": variant_bytes, + "compression_ratio": r(compression_ratio, 3), "max_backlog_jobs": max_backlog, "max_parallel_jobs": max_parallel, "rss_baseline_mib": r(baseline_kb / 1024 if baseline_kb else None, 1), @@ -637,6 +762,11 @@ def build_html(evidence): "VmRSS процесса worker во время задания (меньше — экономнее).", scenarios, profiles, series_for("rss_peak_mib"), " MiB", digits=0, lower_better=True, ), + svg_chart( + "Compression overhead: доля encode в pipeline CPU", + "encode / (decode + scale + encode), % (меньше — дешевле сжатие).", + scenarios, profiles, series_for("encode_share_pct"), " %", digits=1, lower_better=True, + ), ] def fmt(value, digits=1, suffix=""): @@ -653,6 +783,8 @@ def build_html(evidence): f"{fmt(run['queue_lag_p95_ms'], 0)}" f"{fmt(run['time_to_ready_p95_ms'], 0)}" f"{fmt(run['processing_p95_ms'], 0)}" + f"{fmt(run['encode_ms_p95'], 0)}" + f"{fmt(run['encode_share_pct'], 1)}" f"{fmt(run['rss_peak_mib'], 0)}" f"{run['max_parallel_jobs']}" f"{run['max_backlog_jobs']}" @@ -679,6 +811,7 @@ def build_html(evidence): f"{scenario}: пиковый RSS eager относится к serial как {ratio:.1f}×." ) balanced = lookup.get(("balanced", "small_many")) + exact = lookup.get(("balanced", "exactly_right")) huge = lookup.get(("balanced", "huge_few")) if balanced and huge and balanced["time_to_ready_p95_ms"] and huge["time_to_ready_p95_ms"]: observations.append( @@ -687,6 +820,15 @@ def build_html(evidence): "больше, чем для small_many: размер изображения меняет стоимость задания, " "а не только число заданий." ) + if exact and huge and exact["encode_share_pct"] is not None and huge["encode_share_pct"] is not None: + observations.append( + "Compression vs scaling (balanced): exactly_right — encode share " + f"{exact['encode_share_pct']:.1f}%, scale p95 {exact['scale_ms_p95']:.0f} ms; " + f"huge_few — encode share {huge['encode_share_pct']:.1f}%, " + f"scale p95 {huge['scale_ms_p95']:.0f} ms. На exactly_right optimized " + "WebP кодируется в native 1280, поэтому это нижняя граница стоимости " + "сжатия; на huge_few к ней добавляется downscale." + ) huge_serial = lookup.get(("serial", "huge_few")) huge_eager = lookup.get(("eager", "huge_few")) if huge_serial and huge_eager and huge_serial["throughput_jobs_per_s"]: @@ -711,10 +853,10 @@ def build_html(evidence): "заданий в работе и в prefetch-окне брокера." ) observations.append( - "Backpressure: при MAX_PENDING_JOBS=3 API принял " - f"{evidence['backpressure']['accepted_before_503']} задания, затем вернул 503 " - f"с Retry-After={evidence['backpressure']['blocked_retry_after']}; после старта " - f"worker очередь восстановилась за {evidence['backpressure']['recovery_seconds']:.1f} с." + "Backpressure: cap=3 выставлен намеренно, поэтому число принятых заданий — " + "следствие настройки, а не открытие. Значимо то, что API вернул 503 " + f"с Retry-After={evidence['backpressure']['blocked_retry_after']}, а после " + f"старта worker очередь восстановилась за {evidence['backpressure']['recovery_seconds']:.1f} с." ) backpressure_rows = "".join( @@ -786,14 +928,14 @@ code {{ background: #f1f5f9; padding: 1px 5px; border-radius: 4px; }} {evidence['environment']['cpu_count']} CPU, Docker {html.escape(evidence['environment']['docker_version'])}.

    -Гипотеза. Маленькие изображения, которые почти не требуют -масштабирования, должны давать высокий throughput и низкий RSS на задание. -Большие изображения требуют декодирования и downscale, поэтому каждое задание -дороже: при том же числе worker-ов очередь растёт, p95 time-to-ready -увеличивается, а пиковый RSS зависит от concurrency сильнее, чем у маленьких -файлов. Одновременно проверяем компромисс: eager уменьшает ожидание, -но повышает потребление памяти; serial экономит память, но ограничивает -throughput. +Гипотеза. Стоимость задания складывается из decode, масштабирования +и сжатия. exactly_right (1280×1280) не масштабирует optimized-вариант, +поэтому показывает нижнюю границу стоимости сжатия; huge_few добавляет +downscale, а small_many почти не масштабируется, но и пикселей в нём +мало. Большие изображения при том же числе worker-ов увеличивают очередь, +p95 time-to-ready и RSS. Одновременно проверяем компромисс: eager +уменьшает ожидание, но повышает память; serial экономит память, но +ограничивает throughput.
    @@ -813,7 +955,6 @@ throughput.
    {len(runs)}
    прогонов матрицы
    {max(run['rss_peak_mib'] or 0 for run in runs):.0f} MiB
    максимальный RSS worker
    -
    {evidence['backpressure']['accepted_before_503']}
    принято до первого 503
    {evidence['backpressure']['recovery_seconds']:.1f} s
    восстановление после backpressure
    @@ -823,11 +964,16 @@ throughput.

    Полная матрица

    - +{''.join(table_rows)}
    ПрофильСценарийJobsJobs/sQueue lag p95, msReady p95, msProcessing p95, msRSS peak, MiBMax parallelMax backlog
    ПрофильСценарийJobsJobs/sQueue lag p95, msReady p95, msProcessing p95, msEncode p95, msEncode share, %RSS peak, MiBMax parallelMax backlog

    Queue lag = processing_started_at - created_at. Time-to-ready = finished_at - created_at. Processing = finished_at - processing_started_at. +Encode = JPEG thumbnail + JPEG crop + WebP optimized; encode share = +encode / (decode + scale + encode) из worker-лога. +Суммы байт вариантов и оригинала остаются в JSON, но не выносятся в заголовок: +синтетический PNG сжимается слишком хорошо, чтобы отношение байт было честной +метрикой эффективности кодека. RSS — VmRSS процесса worker, сэмплирование примерно каждые 100 мс. Max parallel — максимум одновременно обрабатываемых заданий; max backlog — максимум принятых, но ещё не готовых заданий.

    @@ -836,8 +982,10 @@ Max parallel — максимум одновременно обрабатыва

    Backpressure и восстановление

    Перед этим прогоном API пересоздан с MAX_PENDING_JOBS=3, worker -остановлен. Последовательные upload-ы показывают, где заканчивается принятая -работа. Затем worker запускается, принятые задания должны дойти до ready.

    +остановлен. Cap выбран намеренно: важно не число принятых заданий, а то, что +API возвращает 503 с Retry-After и не теряет уже +подтверждённую работу. Затем worker запускается, принятые задания должны дойти +до ready.

    {backpressure_rows} @@ -857,8 +1005,15 @@ Max parallel — максимум одновременно обрабатыва
    • Один worker, один RabbitMQ и один PostgreSQL: это сравнение настроек, а не HA-кластер. Числа зависят от CPU/диска конкретной машины.
    • -
    • PNG-фикстуры синтетические и хорошо сжимаются; доминирует стоимость декодирования -и масштабирования, а не передача по сети. Размер в байтах указан в JSON.
    • +
    • PNG-фикстуры синтетические и хорошо сжимаются; absolute encode time зависит +от паттерна, но сравнение сценариев при одном паттерне остаётся корректным. +Размер в байтах указан в JSON.
    • +
    • Timing-поля читаются из worker-лога. Это wall-clock внутри процесса, поэтому +при concurrency > 1 в них входит и планировщик, что как раз и нужно для +сравнения профилей.
    • +
    • exactly_right кодирует 1.64 MP WebP, а huge_few после downscale — около +1.23 MP. Сценарий убирает scaling, но не уравнивает объём encode; это указано, +чтобы не сравнивать throughput как «одинаковую работу».
    • RSS сэмплируется из /proc/1/status; короткий прогон не заменяет длительный memory profile и p99.
    • Backpressure cap приблизительный: каждая API-реплика считает pending-задания @@ -879,20 +1034,21 @@ def write_markdown(evidence, path): "", "Synthetic images; no tokens or credentials are stored.", "", - "| Profile | Scenario | Jobs | Jobs/s | Queue lag p95, ms | Ready p95, ms | RSS peak, MiB |", - "| --- | --- | ---: | ---: | ---: | ---: | ---: |", + "| Profile | Scenario | Jobs | Jobs/s | Queue lag p95, ms | Ready p95, ms | Encode p95, ms | Encode share, % | RSS peak, MiB |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", ] for run in evidence["runs"]: lines.append( f"| {run['profile']} | {run['scenario']} | {run['jobs']} | " f"{run['throughput_jobs_per_s']:.2f} | {run['queue_lag_p95_ms']:.0f} | " - f"{run['time_to_ready_p95_ms']:.0f} | {run['rss_peak_mib']:.0f} |" + f"{run['time_to_ready_p95_ms']:.0f} | {run['encode_ms_p95']:.0f} | " + f"{run['encode_share_pct']:.1f} | {run['rss_peak_mib']:.0f} |" ) bp = evidence["backpressure"] lines.extend( [ "", - f"Backpressure: cap={bp['cap']}, accepted={bp['accepted_before_503']}, " + f"Backpressure: cap={bp['cap']} (deliberate), " f"first 503 Retry-After={bp['blocked_retry_after']}, " f"recovery={bp['recovery_seconds']:.2f}s, probe HTTP {bp['probe_status']}.", "", @@ -909,6 +1065,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) parser.add_argument("--small-jobs", type=int, default=30) + parser.add_argument("--exactly-jobs", type=int, default=18) parser.add_argument("--huge-jobs", type=int, default=12) parser.add_argument( "--worker-cpu-limit", @@ -941,9 +1098,14 @@ def main(): client = Client(base_url, timeout=60) alice, _bob, _eve, chat = client.bootstrap() + counts = { + "small_many": args.small_jobs, + "exactly_right": args.exactly_jobs, + "huge_few": args.huge_jobs, + } scenarios = [] for item in SCENARIOS: - count = args.small_jobs if item["name"] == "small_many" else args.huge_jobs + count = counts[item["name"]] data = make_png(item["width"], item["height"]) scenarios.append({**item, "count": count, "bytes": len(data)}) @@ -978,6 +1140,7 @@ def main(): f" throughput={result['throughput_jobs_per_s']} jobs/s " f"queue_p95={result['queue_lag_p95_ms']} ms " f"ready_p95={result['time_to_ready_p95_ms']} ms " + f"encode_share={result['encode_share_pct']}% " f"rss_peak={result['rss_peak_mib']} MiB", flush=True, ) @@ -1046,9 +1209,10 @@ def main(): ), }, "hypothesis": ( - "Small images that fit the variants without large scaling should be cheaper " - "per job; huge images force decode+downscale, so each job costs more memory " - "and time. More concurrency should raise throughput and RSS but reduce queue lag." + "Job cost is decode + scale + encode. exactly_right (1280x1280) keeps the " + "optimized WebP at native size, so it measures the compression floor; " + "huge_few adds downscale, and small_many has few pixels. More concurrency " + "should raise throughput and RSS but reduce queue lag." ), "profiles": PROFILES, "scenarios": scenarios,
    #HTTPLatency, msRetry-AfterError code