From fec2882c9a10ba12b0fb2936745b55509e498e25 Mon Sep 17 00:00:00 2001 From: Mikhail Verkovykh Date: Sun, 15 Nov 2026 00:44:45 +0300 Subject: [PATCH] feat(lab5): track 1 queue experiment and evidence - 3 profiles (serial 1/1, balanced 4/4, eager 8/32) x 2 image sizes - measure throughput, DB-derived queue lag and p95 time-to-ready, worker RSS - separate MAX_PENDING_JOBS=3 backpressure and worker recovery run - self-contained HTML/SVG dashboard plus JSON/Markdown evidence - document methodology, results and limitations in REPORT/README/CHANGELOG --- CHANGELOG.md | 6 +- Makefile | 5 +- README.md | 32 +- REPORT.md | 108 ++- evidence/track1.html | 276 +++++++ evidence/track1.json | 1306 ++++++++++++++++++++++++++++++++++ evidence/track1.md | 16 + scripts/track1_experiment.py | 1070 ++++++++++++++++++++++++++++ 8 files changed, 2809 insertions(+), 10 deletions(-) create mode 100644 evidence/track1.html create mode 100644 evidence/track1.json create mode 100644 evidence/track1.md create mode 100755 scripts/track1_experiment.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 01533cc4..523985d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ - Проверки: async-aware custom tests, негативный случай 42 MP (413), evidence для smoke, отказов, persistence и failover. - `scripts/restart.sh` перезапускает RabbitMQ и worker с сохранением volumes. +- `scripts/track1_experiment.py` и `make track1` — трек 1: матрица + `serial/balanced/eager` × `small_many`/`huge_few`, throughput, queue lag, + p95 time-to-ready, RSS и отдельный backpressure/recovery прогон; + self-contained HTML-дашборд с inline SVG. - Документация: README с запуском и архитектурой, отчёт с таблицей критериев, CHANGELOG. @@ -55,4 +59,4 @@ - `evidence/lab5-smoke.txt`, `evidence/lab5-negative-cases.txt`, `evidence/lab5-grade4.{json,md}`, `evidence/lab5-persistence.{txt}`, `evidence/lab5-persistence-containers.txt`, `evidence/lab5-failover.txt`, - `evidence/reconcile.txt`. + `evidence/reconcile.txt`, `evidence/track1.{json,html,md}`. diff --git a/Makefile b/Makefile index 5d86d171..f1ccbd1c 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover track1 help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,6 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +track1: + $(PYTHON) scripts/track1_experiment.py --base-url "$(BASE_URL)" diff --git a/README.md b/README.md index d02bda57..bc3f00c6 100644 --- a/README.md +++ b/README.md @@ -136,8 +136,9 @@ byte-for-byte и сохраняет исходный SHA-256. Метаданны make check make test BASE_URL=http://localhost:8080 # публичный smoke python3 tests/custom/attachments.py --base-url ... # 413/415/404/400, без 5xx -python3 tests/custom/attachment_flow.py --base-url ... # сквозной upload/download python3 tests/custom/concurrent_uploads.py --base-url ... # конкурентные upload +# attachment_flow.py — двухшаговый helper для verify-attachment-failover.sh, +# а не самостоятельный тест: ему нужны --action upload|download и --state. python3 scripts/lab5_resilience.py --base-url ... # 6 сценариев уровня 4 python3 scripts/track1_experiment.py # трек 1 уровня 5 @@ -151,8 +152,33 @@ python3 scripts/track1_experiment.py # трек 1 уров Трек 1 запускает матрицу профилей `serial/balanced/eager` на маленьких и больших изображениях, измеряет throughput, queue lag, p95 time-to-ready и RSS, -затем отдельно показывает backpressure и восстановление. Результаты и графики -описаны в [REPORT.md](REPORT.md). +затем отдельно показывает backpressure и восстановление. Результаты и графики — +`evidence/track1.{json,html,md}`, разбор — в [REPORT.md](REPORT.md). + +```sh +# Полный прогон: 30 small + 12 huge на каждый профиль, затем backpressure. +make track1 BASE_URL=http://localhost:8080 + +# То же без make. +python3 scripts/track1_experiment.py --base-url http://localhost:8080 + +# Быстрая проверка механики и/или свои лимиты worker-а. +python3 scripts/track1_experiment.py --small-jobs 5 --huge-jobs 3 --skip-backpressure +TRACK1_WORKER_CPU_LIMIT=2.0 TRACK1_WORKER_MEM_LIMIT=3G \ + python3 scripts/track1_experiment.py --base-url http://localhost:8080 +``` + +Эксперимент пересоздаёт worker с `WORKER_CONCURRENCY`/`WORKER_PREFETCH`, +поднимает memory limit до 2 GiB (чтобы `eager` не падал по OOM на больших +изображениях), а в `finally` возвращает конфигурацию из `.env`. Он управляет +только локальным Compose-стендом и не предназначен для production. + +Кратко: на 192×128 eager дал 13.86 jobs/s против 6.88 у serial (2×) при RSS +28→33 MiB. На 2400×1800 все профили дали ~0.45 jobs/s (worker ограничен 1 CPU), +но RSS вырос 205→1122 MiB, а queue lag p95 упал 23.3→17.5 s: concurrency меняет +latency/ресурсы, а не throughput CPU-bound задач. При `MAX_PENDING_JOBS=3` +четвёртый upload получил `503` + `Retry-After: 5`; после старта worker три +принятых задания стали ready за 7.6 s. ## Ограничения diff --git a/REPORT.md b/REPORT.md index 13041c3a..ed8d1404 100644 --- a/REPORT.md +++ b/REPORT.md @@ -115,7 +115,7 @@ backoff ±50% jitter до `JOB_MAX_ATTEMPTS`. После исчерпания | Очистка staging после фиксации | сценарии `worker_stopped`, `crash_after_effect` | `staging_cleared: true`; reconciler удаляет только брошенное | `evidence/lab5-grade4.json`, `evidence/reconcile.txt` | | Негативные случаи и лимиты | `python3 tests/custom/attachments.py` | 413 по bytes и по 42 MP, 415 corrupt/spoof, 404/400, нет 5xx | `evidence/lab5-negative-cases.txt` | | Failover API | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | 118 успешных записей, 1 переходная ошибка, восстановление 2.659 s; остановлена именно `api-2` | `evidence/lab5-failover.txt` | -| Трек 1, уровень 5 | `python3 scripts/track1_experiment.py` | см. раздел «Эксперимент трека 1» ниже | `evidence/track1.json`, `evidence/track1.html` | +| Трек 1, уровень 5 | `python3 scripts/track1_experiment.py` | 3 профиля × 2 размера; eager/small 13.86 jobs/s против serial 6.88; huge_few ~0.45 jobs/s у всех, RSS 205→1122 MiB; cap=3 дал 503 + Retry-After 5 и восстановление за 7.596 s | `evidence/track1.json`, `evidence/track1.html` | ## Эксперименты и отказы @@ -141,10 +141,108 @@ HAProxy; отказ БД или брокера в этом сценарии не ## Эксперимент трека 1 (уровень 5) -Будет заполнено после запуска `scripts/track1_experiment.py`: гипотеза, профили, -размеры изображений, throughput, queue lag, p95 time-to-ready, RSS, -backpressure и восстановление. Сырые данные — `evidence/track1.json`, графики — -`evidence/track1.html`. +### Гипотеза + +Маленькие изображения, которые не требуют масштабирования, дешевле на задание: +выше throughput и ниже RSS. Большие изображения (2400×1800) требуют декодирования +и downscale до 1280×960, поэтому очередь растёт, p95 time-to-ready увеличивается, +а память сильнее зависит от concurrency. Дополнительно проверялся компромисс +«latency против ресурсов»: больше concurrency/prefetch должно снижать queue lag, +но повышать RSS. + +### Методика + +- Профили: `serial` 1/1, `balanced` 4/4, `eager` 8/32 + (concurrency/prefetch). +- Сценарии: `small_many` — 30 PNG 192×128 (941 B, варианты не масштабируются); + `huge_few` — 12 PNG 2400×1800 (53 135 B, 4.32 MP, optimized уменьшается до + 1280×960). +- Для каждой из 6 ячеек worker пересоздавался с профилем, поэтому RSS не + переносился из предыдущего прогона. Изображения загружались параллельно + (до 8 HTTP-клиентов). Все задания дожидались терминального состояния. +- Метрики берутся из PostgreSQL по `attachment_jobs`: queue lag = + `processing_started_at - created_at`, time-to-ready = + `finished_at - created_at`, processing = `finished_at - processing_started_at`. + Throughput = число ready / (последний `finished_at` − первый `created_at`). +- RSS — `VmRSS` процесса worker из `/proc/1/status`, сэмпл примерно каждые + 100 мс; baseline снимался до загрузки. +- Worker ограничен **1 CPU**. На время эксперимента memory limit поднят до + **2 GiB** (в `.env` по умолчанию 1 GiB), иначе `eager/huge_few` упирался бы в + OOM, а не в измеряемый компромисс. Это зафиксировано в JSON. +- Backpressure: API пересоздан с `MAX_PENDING_JOBS=3`, worker остановлен, + upload-ы идут последовательно до первого `503`; затем worker запускается и + проверяется восстановление принятых заданий и новый probe upload. + +Команда: + +```sh +make track1 BASE_URL=http://localhost:8080 +# или +python3 scripts/track1_experiment.py --base-url http://localhost:8080 +``` + +Сырые данные: `evidence/track1.json`; графики: `evidence/track1.html`; +краткая таблица: `evidence/track1.md`. + +### Результаты + +| Профиль | Сценарий | Jobs/s | Queue lag p95, ms | Ready p95, ms | Processing p95, ms | RSS peak, MiB | Max parallel | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| serial | small_many | 6.875 | 3 194 | 3 284 | 113 | 28.2 | 1 | +| serial | huge_few | 0.443 | 23 333 | 25 521 | 2 308 | 204.7 | 1 | +| balanced | small_many | 11.590 | 1 624 | 1 736 | 209 | 30.6 | 4 | +| balanced | huge_few | 0.454 | 17 716 | 25 785 | 25 374 | 728.7 | 4 | +| eager | small_many | 13.858 | 898 | 1 158 | 284 | 32.7 | 8 | +| eager | huge_few | 0.459 | 17 538 | 25 699 | 24 979 | 1 122.1 | 8 | + +Backpressure: cap `3`; первые 3 upload-а получили `202`, четвёртый — `503` с +`Retry-After: 5` и `error_code=unavailable`. После старта worker три принятых +задания стали `ready` за **7.596 s** (включая запуск контейнера и healthcheck), +после чего probe upload получил `202` и дошёл до `ready`. + +### Выводы + +1. **Размер изображения важнее числа заданий.** При `balanced` маленькие файлы + дают 11.59 jobs/s, большие — 0.454 jobs/s (в 25.5× меньше); p95 time-to-ready + растёт с 1 736 до 25 785 ms (в 14.9×). Processing p50 для huge — 8 460 ms + против 211 ms для small: доминирует декодирование/масштабирование, а не + ожидание в очереди. +2. **Eager хорошо работает на маленьких заданиях.** 13.86 jobs/s против 6.88 у + serial (2.0×), queue lag p95 падает с 3 194 до 898 ms, RSS растёт всего с + 28.2 до 32.7 MiB. Это ожидаемый выигрыш от перекрытия лёгких задач. +3. **На huge_few concurrency не даёт throughput.** Все три профиля дают + ~0.44–0.46 jobs/s: worker упирается в 1 CPU, а декодирование и WebP-кодирование + уже насыщают его. Queue lag p95 снижается (23.3 → 17.5 s), но p95 + time-to-ready остаётся ~25.5 s: последние задания всё равно ждут своей доли + CPU. При этом RSS растёт с 205 MiB (serial) до 729 MiB (balanced) и + 1 122 MiB (eager). **Больше concurrency здесь — не бесплатное ускорение, а + рост памяти при том же throughput.** +4. **Backpressure ограничивает принятую работу.** При cap=3 API перестал + принимать новые файлы до чтения тела и вернул retryable `503`; уже + подтверждённые задания не потерялись и завершились после возврата worker. + Это и есть требуемое «bounded acceptance + recovery». +5. **Практический выбор.** Для лёгких изображений разумен `balanced`/`eager`; + для тяжёлых при одном CPU лучше `balanced` (или `serial`, если важна память): + throughput тот же, а RSS в 1.5–5.5× ниже, чем у eager. `prefetch` больше + `concurrency` лишь держит сообщения у worker-а и не увеличивает параллельную + обработку. + +### Ограничения эксперимента + +- Один worker, один RabbitMQ, один PostgreSQL; это сравнение настроек, не HA и + не горизонтальное масштабирование. +- Worker ограничен 1 CPU и 2 GiB; на другом железе абсолютные числа и точка + насыщения будут другими. Относительный вывод «CPU-bound huge не ускоряется + concurrency» получен именно для этого лимита. +- PNG-фикстуры синтетические и хорошо сжимаются; доминирует стоимость + декодирования/масштабирования, а не сеть. Размеры в байтах и SHA-256 есть в + JSON. +- Один прогон на ячейку, без длительного прогрева и повторений; p95 по 12 + заданиям чувствителен к выбросу. Это учебная оценка, не SLA. +- `MAX_PENDING_JOBS` проверяется приблизительно и не одной транзакцией с + вставкой; при нескольких API-репликах возможен небольшой перебор. +- Время восстановления backpressure включает `docker compose start` и + healthcheck, поэтому это верхняя оценка, а не время чистой обработки. ## Ограничения и следующие шаги diff --git a/evidence/track1.html b/evidence/track1.html new file mode 100644 index 00000000..53e7638b --- /dev/null +++ b/evidence/track1.html @@ -0,0 +1,276 @@ + + + + + +Lab5 track 1 — очередь под перегрузкой + + + +
+

Lab5, трек 1: очередь под перегрузкой

+

Сгенерировано 2026-11-14T21:42:03.122104+00:00. Стенд: http://172.18.0.1:8080, +4 CPU, Docker 5.6.1.

+ +
+Гипотеза. Маленькие изображения, которые почти не требуют +масштабирования, должны давать высокий throughput и низкий RSS на задание. +Большие изображения требуют декодирования и downscale, поэтому каждое задание +дороже: при том же числе worker-ов очередь растёт, p95 time-to-ready +увеличивается, а пиковый RSS зависит от concurrency сильнее, чем у маленьких +файлов. Одновременно проверяем компромисс: eager уменьшает ожидание, +но повышает потребление памяти; serial экономит память, но ограничивает +throughput. +
+ +
+

Профили и сценарии

+ + + + + +
ПрофильConcurrencyPrefetchСмысл
serial11one job at a time; minimum memory, maximum queueing
balanced44moderate in-process parallelism and broker window
eager832wide broker window; highest memory, lowest latency
+

Сценарии: +small_many — 192×128, 30 jobs, many small images; variants fit or crop without large scaling; huge_few — 2400×1800, 12 jobs, few 4.32 MP images; optimized variant must scale down to 1280.

+
+ +
+

Ключевые числа

+
+
6
прогонов матрицы
+
1122 MiB
максимальный RSS worker
+
3
принято до первого 503
+
7.6 s
восстановление после backpressure
+
+
+ +
+Throughput: готовые задания в секунду +Чем выше, тем быстрее pipeline обрабатывает очередь. + +0 jobs/s + +4 jobs/s + +8 jobs/s + +12 jobs/s + +16 jobs/s + +6.88 jobs/s + +11.59 jobs/s + +13.86 jobs/s +small_many + +0.44 jobs/s + +0.45 jobs/s + +0.46 jobs/s +huge_few + +serial + +balanced + +eager +higher is better +
+
+p95 time-to-ready: от 202 до ready +95% заданий стали ready быстрее этого времени (меньше — лучше). + +0 ms + +7606 ms + +15213 ms + +22819 ms + +30426 ms + +3284 ms + +1736 ms + +1158 ms +small_many + +25521 ms + +25785 ms + +25699 ms +huge_few + +serial + +balanced + +eager +lower is better +
+
+p95 queue lag: ожидание до старта обработки +Время между durable 202 и первым claim worker-а (меньше — лучше). + +0 ms + +6883 ms + +13766 ms + +20650 ms + +27533 ms + +3194 ms + +1624 ms + +898 ms +small_many + +23333 ms + +17716 ms + +17538 ms +huge_few + +serial + +balanced + +eager +lower is better +
+
+Пиковый RSS worker +VmRSS процесса worker во время задания (меньше — экономнее). + +0 MiB + +331 MiB + +662 MiB + +993 MiB + +1324 MiB + +28 MiB + +31 MiB + +33 MiB +small_many + +205 MiB + +729 MiB + +1122 MiB +huge_few + +serial + +balanced + +eager +lower is better +
+ +
+

Полная матрица

+ + + +
ПрофильСценарийJobsJobs/sQueue lag p95, msReady p95, msProcessing p95, msRSS peak, MiBMax parallelMax backlog
serialsmall_many306.883194328415228127
serialhuge_few120.4423333255212311205112
balancedsmall_many3011.591624173633631426
balancedhuge_few120.4517716257858710729412
eagersmall_many3013.86898115856633826
eagerhuge_few120.461753825699173201122812
+

Queue lag = processing_started_at - created_at. +Time-to-ready = finished_at - created_at. Processing = finished_at - processing_started_at. +RSS — VmRSS процесса worker, сэмплирование примерно каждые 100 мс. +Max parallel — максимум одновременно обрабатываемых заданий; max backlog — +максимум принятых, но ещё не готовых заданий.

+
+ +
+

Backpressure и восстановление

+

Перед этим прогоном API пересоздан с MAX_PENDING_JOBS=3, worker +остановлен. Последовательные upload-ы показывают, где заканчивается принятая +работа. Затем worker запускается, принятые задания должны дойти до ready.

+ + + +
#HTTPLatency, msRetry-AfterError code
120218.2——
220217.4——
320210.8——
45033.95unavailable
+

Recovery: 3 принятых заданий +стали ready за 7.60 с после старта worker. +Финальный probe upload: HTTP 202 → ready.

+
+ +
+

Что это значит

+
  • small_many: eager быстрее serial в 2.0× по throughput.
  • small_many: пиковый RSS eager относится к serial как 1.2×.
  • huge_few: throughput eager и serial практически одинаков.
  • huge_few: пиковый RSS eager относится к serial как 5.5×.
  • При balanced p95 time-to-ready для huge_few в 14.9× больше, чем для small_many: размер изображения меняет стоимость задания, а не только число заданий.
  • На huge_few eager почти не поднял throughput (1.04×): worker ограничен 1.0 CPU, а декодирование и масштабирование упираются в CPU. Concurrency снижает queue lag, но при этом растёт RSS; это и есть компромисс latency/ресурсы, а не бесплатное ускорение.
  • serial ограничивает одновременно обрабатываемые задания одним, поэтому queue lag растёт, но RSS остаётся минимальным. eager держит больше заданий в работе и в prefetch-окне брокера.
  • Backpressure: при MAX_PENDING_JOBS=3 API принял 3 задания, затем вернул 503 с Retry-After=5; после старта worker очередь восстановилась за 7.6 с.
+
+ +
+

Ограничения

+
    +
  • Один worker, один RabbitMQ и один PostgreSQL: это сравнение настроек, а не +HA-кластер. Числа зависят от CPU/диска конкретной машины.
  • +
  • PNG-фикстуры синтетические и хорошо сжимаются; доминирует стоимость декодирования +и масштабирования, а не передача по сети. Размер в байтах указан в JSON.
  • +
  • RSS сэмплируется из /proc/1/status; короткий прогон не заменяет +длительный memory profile и p99.
  • +
  • Backpressure cap приблизительный: каждая API-реплика считает pending-задания +по общей БД, но проверка и вставка не являются одной транзакцией.
  • +
+
+
+ + diff --git a/evidence/track1.json b/evidence/track1.json new file mode 100644 index 00000000..d9adae62 --- /dev/null +++ b/evidence/track1.json @@ -0,0 +1,1306 @@ +{ + "generated_at": "2026-11-14T21:42:03.122104+00:00", + "environment": { + "base_url": "http://172.18.0.1:8080", + "host": "48f08d2e9267", + "platform": "Linux-6.12.73+deb13-amd64-x86_64-with-glibc2.39", + "cpu_count": 4, + "python": "3.12.3", + "docker_version": "5.6.1", + "compose_project": "mtusi2026lab", + "worker_container": "mtusi2026lab-worker-1", + "worker_cpu_limit": "1.0", + "worker_mem_limit": "2G", + "worker_cpu_limit_default": "1.0", + "worker_mem_limit_default": "1G", + "note": "Experimental workers use a larger memory limit than .env so the eager huge-image run can hold decoded images without OOM; CPU is still limited so throughput is not a host-CPU artifact." + }, + "hypothesis": "Small images that fit the variants without large scaling should be cheaper per job; huge images force decode+downscale, so each job costs more memory and time. More concurrency should raise throughput and RSS but reduce queue lag.", + "profiles": [ + { + "name": "serial", + "concurrency": 1, + "prefetch": 1, + "note": "one job at a time; minimum memory, maximum queueing" + }, + { + "name": "balanced", + "concurrency": 4, + "prefetch": 4, + "note": "moderate in-process parallelism and broker window" + }, + { + "name": "eager", + "concurrency": 8, + "prefetch": 32, + "note": "wide broker window; highest memory, lowest latency" + } + ], + "scenarios": [ + { + "name": "small_many", + "width": 192, + "height": 128, + "description": "many small images; variants fit or crop without large scaling", + "count": 30, + "bytes": 941 + }, + { + "name": "huge_few", + "width": 2400, + "height": 1800, + "description": "few 4.32 MP images; optimized variant must scale down to 1280", + "count": 12, + "bytes": 53135 + } + ], + "runs": [ + { + "profile": "serial", + "scenario": "small_many", + "concurrency": 1, + "prefetch": 1, + "jobs": 30, + "image_width": 192, + "image_height": 128, + "image_bytes": 941, + "image_sha256": "7e90807e4df32b24ab7e0f8f540c76b93ed2a0e9f91e7b3350748cfc8029a088", + "throughput_jobs_per_s": 6.875, + "pipeline_span_seconds": 4.364, + "submit_seconds": 0.987, + "wall_seconds": 4.922, + "upload_p50_ms": 43.4, + "upload_p95_ms": 99.8, + "queue_lag_p50_ms": 2030.8, + "queue_lag_p95_ms": 3193.7, + "queue_lag_max_ms": 3385.0, + "time_to_ready_p50_ms": 2119.7, + "time_to_ready_p95_ms": 3284.1, + "time_to_ready_max_ms": 3475.8, + "processing_p50_ms": 85.7, + "processing_p95_ms": 151.5, + "max_backlog_jobs": 27, + "max_parallel_jobs": 1, + "rss_baseline_mib": 20.3, + "rss_peak_mib": 28.2, + "rss_end_mib": 27.1, + "rss_samples": 63, + "rss_delta_mib": 7.9, + "timeline": [ + { + "t": 1.356, + "queued": 24, + "processing": 0, + "ready": 6, + "failed": 0 + }, + { + "t": 1.998, + "queued": 19, + "processing": 1, + "ready": 10, + "failed": 0 + }, + { + "t": 2.711, + "queued": 15, + "processing": 0, + "ready": 15, + "failed": 0 + }, + { + "t": 3.265, + "queued": 10, + "processing": 1, + "ready": 19, + "failed": 0 + }, + { + "t": 3.754, + "queued": 5, + "processing": 1, + "ready": 24, + "failed": 0 + }, + { + "t": 4.384, + "queued": 0, + "processing": 1, + "ready": 29, + "failed": 0 + }, + { + "t": 4.922, + "queued": 0, + "processing": 0, + "ready": 30, + "failed": 0 + } + ] + }, + { + "profile": "serial", + "scenario": "huge_few", + "concurrency": 1, + "prefetch": 1, + "jobs": 12, + "image_width": 2400, + "image_height": 1800, + "image_bytes": 53135, + "image_sha256": "68c88fcbe5c4ba31ee745047e8be4dc64e5d7711f81f5e50a4276fb52cf925ba", + "throughput_jobs_per_s": 0.443, + "pipeline_span_seconds": 27.106, + "submit_seconds": 0.52, + "wall_seconds": 27.417, + "upload_p50_ms": 162.6, + "upload_p95_ms": 227.5, + "queue_lag_p50_ms": 12336.3, + "queue_lag_p95_ms": 23333.0, + "queue_lag_max_ms": 24511.6, + "time_to_ready_p50_ms": 14530.2, + "time_to_ready_p95_ms": 25521.0, + "time_to_ready_max_ms": 26776.2, + "processing_p50_ms": 2233.0, + "processing_p95_ms": 2311.1, + "max_backlog_jobs": 12, + "max_parallel_jobs": 1, + "rss_baseline_mib": 20.5, + "rss_peak_mib": 204.7, + "rss_end_mib": 203.1, + "rss_samples": 278, + "rss_delta_mib": 184.2, + "timeline": [ + { + "t": 0.827, + "queued": 11, + "processing": 1, + "ready": 0, + "failed": 0 + }, + { + "t": 1.494, + "queued": 11, + "processing": 1, + "ready": 0, + "failed": 0 + }, + { + "t": 2.002, + "queued": 11, + "processing": 1, + "ready": 0, + "failed": 0 + }, + { + "t": 2.488, + "queued": 10, + "processing": 1, + "ready": 1, + "failed": 0 + }, + { + "t": 3.193, + "queued": 10, + "processing": 1, + "ready": 1, + "failed": 0 + }, + { + "t": 4.011, + "queued": 10, + "processing": 1, + "ready": 1, + "failed": 0 + }, + { + "t": 4.519, + "queued": 10, + "processing": 1, + "ready": 1, + "failed": 0 + }, + { + "t": 5.153, + "queued": 9, + "processing": 1, + "ready": 2, + "failed": 0 + }, + { + "t": 5.723, + "queued": 9, + "processing": 1, + "ready": 2, + "failed": 0 + }, + { + "t": 6.187, + "queued": 9, + "processing": 1, + "ready": 2, + "failed": 0 + }, + { + "t": 6.671, + "queued": 9, + "processing": 1, + "ready": 2, + "failed": 0 + }, + { + "t": 7.283, + "queued": 8, + "processing": 1, + "ready": 3, + "failed": 0 + }, + { + "t": 7.94, + "queued": 8, + "processing": 1, + "ready": 3, + "failed": 0 + }, + { + "t": 8.452, + "queued": 8, + "processing": 1, + "ready": 3, + "failed": 0 + }, + { + "t": 9.103, + "queued": 8, + "processing": 1, + "ready": 3, + "failed": 0 + }, + { + "t": 9.694, + "queued": 7, + "processing": 1, + "ready": 4, + "failed": 0 + }, + { + "t": 10.185, + "queued": 7, + "processing": 1, + "ready": 4, + "failed": 0 + }, + { + "t": 10.674, + "queued": 7, + "processing": 1, + "ready": 4, + "failed": 0 + }, + { + "t": 11.333, + "queued": 7, + "processing": 1, + "ready": 4, + "failed": 0 + }, + { + "t": 11.836, + "queued": 6, + "processing": 1, + "ready": 5, + "failed": 0 + }, + { + "t": 12.386, + "queued": 6, + "processing": 1, + "ready": 5, + "failed": 0 + }, + { + "t": 13.112, + "queued": 6, + "processing": 1, + "ready": 5, + "failed": 0 + }, + { + "t": 13.692, + "queued": 6, + "processing": 1, + "ready": 5, + "failed": 0 + }, + { + "t": 14.188, + "queued": 5, + "processing": 1, + "ready": 6, + "failed": 0 + }, + { + "t": 14.684, + "queued": 5, + "processing": 1, + "ready": 6, + "failed": 0 + }, + { + "t": 15.277, + "queued": 5, + "processing": 1, + "ready": 6, + "failed": 0 + }, + { + "t": 15.82, + "queued": 5, + "processing": 1, + "ready": 6, + "failed": 0 + }, + { + "t": 16.477, + "queued": 4, + "processing": 1, + "ready": 7, + "failed": 0 + }, + { + "t": 17.103, + "queued": 4, + "processing": 1, + "ready": 7, + "failed": 0 + }, + { + "t": 17.622, + "queued": 4, + "processing": 1, + "ready": 7, + "failed": 0 + }, + { + "t": 18.203, + "queued": 4, + "processing": 1, + "ready": 7, + "failed": 0 + }, + { + "t": 18.738, + "queued": 3, + "processing": 1, + "ready": 8, + "failed": 0 + }, + { + "t": 19.242, + "queued": 3, + "processing": 1, + "ready": 8, + "failed": 0 + }, + { + "t": 19.806, + "queued": 3, + "processing": 1, + "ready": 8, + "failed": 0 + }, + { + "t": 20.406, + "queued": 3, + "processing": 1, + "ready": 8, + "failed": 0 + }, + { + "t": 20.89, + "queued": 2, + "processing": 1, + "ready": 9, + "failed": 0 + }, + { + "t": 21.44, + "queued": 2, + "processing": 1, + "ready": 9, + "failed": 0 + }, + { + "t": 22.162, + "queued": 2, + "processing": 1, + "ready": 9, + "failed": 0 + }, + { + "t": 22.773, + "queued": 2, + "processing": 1, + "ready": 9, + "failed": 0 + }, + { + "t": 23.257, + "queued": 1, + "processing": 1, + "ready": 10, + "failed": 0 + }, + { + "t": 23.854, + "queued": 1, + "processing": 1, + "ready": 10, + "failed": 0 + }, + { + "t": 24.452, + "queued": 1, + "processing": 1, + "ready": 10, + "failed": 0 + }, + { + "t": 24.927, + "queued": 1, + "processing": 1, + "ready": 10, + "failed": 0 + }, + { + "t": 25.472, + "queued": 0, + "processing": 1, + "ready": 11, + "failed": 0 + }, + { + "t": 26.224, + "queued": 0, + "processing": 1, + "ready": 11, + "failed": 0 + }, + { + "t": 26.95, + "queued": 0, + "processing": 1, + "ready": 11, + "failed": 0 + }, + { + "t": 27.417, + "queued": 0, + "processing": 0, + "ready": 12, + "failed": 0 + } + ] + }, + { + "profile": "balanced", + "scenario": "small_many", + "concurrency": 4, + "prefetch": 4, + "jobs": 30, + "image_width": 192, + "image_height": 128, + "image_bytes": 941, + "image_sha256": "7e90807e4df32b24ab7e0f8f540c76b93ed2a0e9f91e7b3350748cfc8029a088", + "throughput_jobs_per_s": 11.59, + "pipeline_span_seconds": 2.588, + "submit_seconds": 1.049, + "wall_seconds": 2.966, + "upload_p50_ms": 58.0, + "upload_p95_ms": 178.8, + "queue_lag_p50_ms": 1131.4, + "queue_lag_p95_ms": 1623.9, + "queue_lag_max_ms": 1647.8, + "time_to_ready_p50_ms": 1335.0, + "time_to_ready_p95_ms": 1735.8, + "time_to_ready_max_ms": 1756.1, + "processing_p50_ms": 211.1, + "processing_p95_ms": 336.3, + "max_backlog_jobs": 26, + "max_parallel_jobs": 4, + "rss_baseline_mib": 20.5, + "rss_peak_mib": 30.6, + "rss_end_mib": 28.5, + "rss_samples": 43, + "rss_delta_mib": 10.1, + "timeline": [ + { + "t": 1.679, + "queued": 14, + "processing": 4, + "ready": 12, + "failed": 0 + }, + { + "t": 2.514, + "queued": 4, + "processing": 2, + "ready": 24, + "failed": 0 + }, + { + "t": 2.966, + "queued": 0, + "processing": 0, + "ready": 30, + "failed": 0 + } + ] + }, + { + "profile": "balanced", + "scenario": "huge_few", + "concurrency": 4, + "prefetch": 4, + "jobs": 12, + "image_width": 2400, + "image_height": 1800, + "image_bytes": 53135, + "image_sha256": "68c88fcbe5c4ba31ee745047e8be4dc64e5d7711f81f5e50a4276fb52cf925ba", + "throughput_jobs_per_s": 0.454, + "pipeline_span_seconds": 26.412, + "submit_seconds": 0.688, + "wall_seconds": 26.671, + "upload_p50_ms": 178.3, + "upload_p95_ms": 298.3, + "queue_lag_p50_ms": 8712.6, + "queue_lag_p95_ms": 17716.4, + "queue_lag_max_ms": 18009.6, + "time_to_ready_p50_ms": 17199.1, + "time_to_ready_p95_ms": 25784.6, + "time_to_ready_max_ms": 25919.7, + "processing_p50_ms": 8459.9, + "processing_p95_ms": 8709.7, + "max_backlog_jobs": 12, + "max_parallel_jobs": 4, + "rss_baseline_mib": 20.4, + "rss_peak_mib": 728.7, + "rss_end_mib": 609.9, + "rss_samples": 254, + "rss_delta_mib": 708.3, + "timeline": [ + { + "t": 1.028, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 1.694, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 2.319, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 2.905, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 3.537, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 4.173, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 4.829, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 5.508, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 6.11, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 6.603, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 7.14, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 7.796, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 8.345, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 8.854, + "queued": 8, + "processing": 4, + "ready": 0, + "failed": 0 + }, + { + "t": 9.667, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 10.488, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 11.103, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 11.769, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 12.384, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 12.857, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 13.371, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 14.228, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 14.929, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 15.574, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 16.234, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 16.711, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 17.257, + "queued": 4, + "processing": 4, + "ready": 4, + "failed": 0 + }, + { + "t": 17.945, + "queued": 2, + "processing": 3, + "ready": 7, + "failed": 0 + }, + { + "t": 18.674, + "queued": 1, + "processing": 3, + "ready": 8, + "failed": 0 + }, + { + "t": 19.251, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 19.997, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 20.662, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 21.146, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 21.758, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 22.395, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 22.945, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 23.693, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 24.381, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 24.906, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 25.597, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 26.174, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 26.671, + "queued": 0, + "processing": 0, + "ready": 12, + "failed": 0 + } + ] + }, + { + "profile": "eager", + "scenario": "small_many", + "concurrency": 8, + "prefetch": 32, + "jobs": 30, + "image_width": 192, + "image_height": 128, + "image_bytes": 941, + "image_sha256": "7e90807e4df32b24ab7e0f8f540c76b93ed2a0e9f91e7b3350748cfc8029a088", + "throughput_jobs_per_s": 13.858, + "pipeline_span_seconds": 2.165, + "submit_seconds": 1.168, + "wall_seconds": 2.318, + "upload_p50_ms": 46.4, + "upload_p95_ms": 126.7, + "queue_lag_p50_ms": 749.2, + "queue_lag_p95_ms": 897.8, + "queue_lag_max_ms": 1052.1, + "time_to_ready_p50_ms": 1013.5, + "time_to_ready_p95_ms": 1157.8, + "time_to_ready_max_ms": 1175.0, + "processing_p50_ms": 271.8, + "processing_p95_ms": 565.9, + "max_backlog_jobs": 26, + "max_parallel_jobs": 8, + "rss_baseline_mib": 20.4, + "rss_peak_mib": 32.7, + "rss_end_mib": 28.8, + "rss_samples": 37, + "rss_delta_mib": 12.3, + "timeline": [ + { + "t": 1.637, + "queued": 10, + "processing": 6, + "ready": 14, + "failed": 0 + }, + { + "t": 2.318, + "queued": 0, + "processing": 0, + "ready": 30, + "failed": 0 + } + ] + }, + { + "profile": "eager", + "scenario": "huge_few", + "concurrency": 8, + "prefetch": 32, + "jobs": 12, + "image_width": 2400, + "image_height": 1800, + "image_bytes": 53135, + "image_sha256": "68c88fcbe5c4ba31ee745047e8be4dc64e5d7711f81f5e50a4276fb52cf925ba", + "throughput_jobs_per_s": 0.459, + "pipeline_span_seconds": 26.164, + "submit_seconds": 0.621, + "wall_seconds": 26.514, + "upload_p50_ms": 211.1, + "upload_p95_ms": 334.8, + "queue_lag_p50_ms": 603.5, + "queue_lag_p95_ms": 17537.9, + "queue_lag_max_ms": 17742.4, + "time_to_ready_p50_ms": 17932.3, + "time_to_ready_p95_ms": 25698.9, + "time_to_ready_max_ms": 25761.3, + "processing_p50_ms": 16723.3, + "processing_p95_ms": 17319.7, + "max_backlog_jobs": 12, + "max_parallel_jobs": 8, + "rss_baseline_mib": 20.0, + "rss_peak_mib": 1122.1, + "rss_end_mib": 763.8, + "rss_samples": 254, + "rss_delta_mib": 1102.1, + "timeline": [ + { + "t": 1.084, + "queued": 6, + "processing": 6, + "ready": 0, + "failed": 0 + }, + { + "t": 1.861, + "queued": 5, + "processing": 7, + "ready": 0, + "failed": 0 + }, + { + "t": 2.395, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 3.195, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 3.804, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 4.307, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 4.914, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 5.49, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 5.99, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 6.547, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 7.386, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 8.066, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 8.621, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 9.226, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 9.729, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 10.222, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 10.837, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 11.613, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 12.158, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 12.822, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 13.409, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 13.899, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 14.425, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 15.051, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 15.644, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 16.265, + "queued": 4, + "processing": 8, + "ready": 0, + "failed": 0 + }, + { + "t": 17.048, + "queued": 4, + "processing": 7, + "ready": 1, + "failed": 0 + }, + { + "t": 17.714, + "queued": 3, + "processing": 5, + "ready": 4, + "failed": 0 + }, + { + "t": 18.216, + "queued": 1, + "processing": 6, + "ready": 5, + "failed": 0 + }, + { + "t": 18.891, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 19.529, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 20.017, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 20.748, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 21.529, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 22.067, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 22.652, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 23.251, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 23.924, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 24.548, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 25.071, + "queued": 0, + "processing": 4, + "ready": 8, + "failed": 0 + }, + { + "t": 25.888, + "queued": 0, + "processing": 3, + "ready": 9, + "failed": 0 + }, + { + "t": 26.514, + "queued": 0, + "processing": 0, + "ready": 12, + "failed": 0 + } + ] + } + ], + "backpressure": { + "cap": 3, + "attempts": [ + { + "index": 0, + "status": 202, + "elapsed_ms": 18.2, + "retry_after": null, + "error_code": null + }, + { + "index": 1, + "status": 202, + "elapsed_ms": 17.4, + "retry_after": null, + "error_code": null + }, + { + "index": 2, + "status": 202, + "elapsed_ms": 10.8, + "retry_after": null, + "error_code": null + }, + { + "index": 3, + "status": 503, + "elapsed_ms": 3.9, + "retry_after": "5", + "error_code": "unavailable" + } + ], + "accepted_before_503": 3, + "blocked_status": 503, + "blocked_retry_after": "5", + "recovery_seconds": 7.596, + "recovered_jobs": 3, + "probe_status": 202, + "probe_ready": true, + "image_bytes": 941, + "image_sha256": "7e90807e4df32b24ab7e0f8f540c76b93ed2a0e9f91e7b3350748cfc8029a088" + } +} diff --git a/evidence/track1.md b/evidence/track1.md new file mode 100644 index 00000000..856c38ce --- /dev/null +++ b/evidence/track1.md @@ -0,0 +1,16 @@ +# Lab5 track 1 — queue under load + +Generated: `2026-11-14T21:42:03.122104+00:00` + +Synthetic images; no tokens or credentials are stored. + +| Profile | Scenario | Jobs | Jobs/s | Queue lag p95, ms | Ready p95, ms | RSS peak, MiB | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| serial | small_many | 30 | 6.88 | 3194 | 3284 | 28 | +| serial | huge_few | 12 | 0.44 | 23333 | 25521 | 205 | +| balanced | small_many | 30 | 11.59 | 1624 | 1736 | 31 | +| balanced | huge_few | 12 | 0.45 | 17716 | 25785 | 729 | +| eager | small_many | 30 | 13.86 | 898 | 1158 | 33 | +| eager | huge_few | 12 | 0.46 | 17538 | 25699 | 1122 | + +Backpressure: cap=3, accepted=3, first 503 Retry-After=5, recovery=7.60s, probe HTTP 202. diff --git a/scripts/track1_experiment.py b/scripts/track1_experiment.py new file mode 100755 index 00000000..56731244 --- /dev/null +++ b/scripts/track1_experiment.py @@ -0,0 +1,1070 @@ +#!/usr/bin/env python3 +"""Lab5 grade-5 track 1: queue behaviour under small vs huge images. + +The experiment is deliberately short and self-contained: + + profiles: serial (concurrency 1, prefetch 1) + balanced (4/4) + eager (8/32) + scenarios: small_many (30 x 192x128 PNG, variants mostly fit without scaling) + huge_few (12 x 2400x1800 PNG, every job needs heavy scaling) + +For each cell the worker is recreated with the profile, the images are uploaded +concurrently, and the script records: + + * throughput (ready jobs / pipeline span), + * queue lag: created_at -> processing_started_at (95th percentile), + * time-to-ready: created_at -> finished_at (p50/p95), + * worker RSS baseline/peak sampled from /proc/1/status, + * upload latency and max in-flight jobs. + +A final backpressure run caps MAX_PENDING_JOBS at 3, stops the worker, shows +503 + Retry-After, then starts the worker and proves recovery. + +Outputs evidence/track1.json plus a self-contained HTML dashboard with inline +SVG (no external JS/CSS/fonts). The script only drives the disposable local +Compose stand. It restores the default worker/API environment in a finally +block, so a failed run does not leave the stand throttled. + +Usage: + python3 scripts/track1_experiment.py --base-url http://localhost:8080 + python3 scripts/track1_experiment.py --base-url http://172.18.0.1:8080 \ + --small-jobs 30 --huge-jobs 12 +""" +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import math +import os +import platform +import struct +import subprocess +import sys +import threading +import time +import zlib +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +sys.path.insert(0, str(ROOT / "scripts")) + +from client import Client, Failure, check # noqa: E402 +from lab5_resilience import ( # noqa: E402 + compose, + compose_capture, + inspect, + recreate_api, + recreate_worker, + restore_api, + restore_worker, + service_ids, + wait_healthy, + wait_stopped, +) + +PROFILES = [ + {"name": "serial", "concurrency": 1, "prefetch": 1, + "note": "one job at a time; minimum memory, maximum queueing"}, + {"name": "balanced", "concurrency": 4, "prefetch": 4, + "note": "moderate in-process parallelism and broker window"}, + {"name": "eager", "concurrency": 8, "prefetch": 32, + "note": "wide broker window; highest memory, lowest latency"}, +] + +SCENARIOS = [ + {"name": "small_many", "width": 192, "height": 128, + "description": "many small images; variants fit or crop without large scaling"}, + {"name": "huge_few", "width": 2400, "height": 1800, + "description": "few 4.32 MP images; optimized variant must scale down to 1280"}, +] + + +# --------------------------------------------------------------------------- +# Fixture generation and small helpers + + +def png_chunk(kind, payload): + crc = zlib.crc32(kind + payload) & 0xFFFFFFFF + return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", crc) + + +# AGENT: fixtures are generated in-process so the experiment has no binary +# fixtures in git and the exact bytes/pixels are recorded in the JSON. +def make_png(width, height): + """Deterministic RGB PNG. The pattern compresses well, but the decoder + still allocates width*height*3 bytes, which is exactly what the experiment + wants to stress without making upload bandwidth the bottleneck.""" + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + base = bytes(range(256)) + row_len = width * 3 + repeated = base * ((row_len // 256) + 2) + compressor = zlib.compressobj(6) + idat = bytearray() + for y in range(height): + offset = (y * 7) % 256 + idat += compressor.compress(b"\x00" + repeated[offset : offset + row_len]) + idat += compressor.flush() + return ( + b"\x89PNG\r\n\x1a\n" + + png_chunk(b"IHDR", ihdr) + + png_chunk(b"IDAT", bytes(idat)) + + png_chunk(b"IEND", b"") + ) + + +def percentile(values, p): + if not values: + return None + ordered = sorted(values) + if len(ordered) == 1: + return float(ordered[0]) + position = (len(ordered) - 1) * (p / 100.0) + low = math.floor(position) + high = math.ceil(position) + if low == high: + return float(ordered[low]) + weight = position - low + return float(ordered[low] * (1 - weight) + ordered[high] * weight) + + +def r(value, digits=2): + return None if value is None else round(float(value), digits) + + +def worker_container(): + ids = service_ids("worker") + if not ids: + raise Failure("worker container is not running") + name = inspect(ids[0]).get("Name", "").lstrip("/") + if not name: + raise Failure("cannot resolve worker container name") + return name + + +def psql(sql): + # lab5_resilience.psql is not exported under that name in all versions; + # keep the small wrapper local so this script is easy to audit. + import shlex + + command = ( + 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -F "|" ' + "-v ON_ERROR_STOP=1 -c " + shlex.quote(sql) + ) + out = compose_capture("exec", "-T", "postgres", "sh", "-c", command) + return [line for line in out.splitlines() if line.strip()] + + +def job_rows(attachment_ids): + if not attachment_ids: + return [] + quoted = ",".join("'" + item + "'" for item in attachment_ids) + sql = ( + "SELECT attachment_id, status, attempt, coalesce(error_code,''), " + "extract(epoch from created_at), " + "coalesce(extract(epoch from processing_started_at), 0), " + "coalesce(extract(epoch from finished_at), 0) " + f"FROM attachment_jobs WHERE attachment_id IN ({quoted})" + ) + rows = [] + for line in psql(sql): + parts = line.split("|") + if len(parts) != 7: + continue + rows.append( + { + "attachment_id": parts[0], + "status": parts[1], + "attempt": int(parts[2]), + "error_code": parts[3] or None, + "created_at": float(parts[4]), + "processing_started_at": float(parts[5]) if parts[5] not in ("", "0") else None, + "finished_at": float(parts[6]) if parts[6] not in ("", "0") else None, + } + ) + return rows + + +def pending_count(): + lines = psql( + "SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')" + ) + return int(lines[0]) if lines else 0 + + +def wait_drained(timeout=180): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = pending_count() + if last == 0: + return + time.sleep(0.5) + raise Failure(f"pipeline did not drain within {timeout}s; pending={last}") + + +def upload_one(base_url, actor, chat_id, data, filename): + client = Client(base_url, timeout=60) + started = time.monotonic() + try: + response = client.call( + "POST", + f"/api/v1/chats/{chat_id}/attachments?filename={filename}", + expected=(202, 503), + actor=actor, + raw=data, + headers={"Content-Type": "image/png"}, + schema="Attachment", + ) + except Failure as exc: + return {"status": 0, "error": str(exc), "elapsed": time.monotonic() - started} + result = { + "status": response["status"], + "elapsed": time.monotonic() - started, + "retry_after": response["headers"].get("Retry-After"), + } + if response["status"] == 202: + result["attachment_id"] = response["json"]["id"] + else: + result["error_code"] = response["json"]["error"]["code"] + return result + + +def upload_many(base_url, actor, chat_id, data, count, prefix): + with ThreadPoolExecutor(max_workers=min(8, count)) as pool: + futures = [ + pool.submit(upload_one, base_url, actor, chat_id, data, f"{prefix}-{index}.png") + for index in range(count) + ] + return [future.result() for future in futures] + + +# --------------------------------------------------------------------------- +# Worker RSS sampler + + +# AGENT: RSS is sampled inside the worker container, not with `docker stats`, +# so the metric is the process VmRSS without host/daemon noise. +class RssSampler: + def __init__(self, container): + self.container = container + self.samples = [] + self._stop = threading.Event() + self._process = None + self._thread = None + + def start(self): + self._process = subprocess.Popen( + [ + "docker", + "exec", + self.container, + "sh", + "-c", + "while :; do awk '/VmRSS/{print $2}' /proc/1/status; sleep 0.1; done", + ], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + self._thread = threading.Thread(target=self._read, daemon=True) + self._thread.start() + + def _read(self): + if not self._process or not self._process.stdout: + return + for line in self._process.stdout: + if self._stop.is_set(): + break + try: + self.samples.append(int(line.strip())) + except ValueError: + continue + + def snapshot(self): + return list(self.samples) + + def stop(self): + self._stop.set() + if self._process: + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + if self._thread: + self._thread.join(timeout=2) + return list(self.samples) + + +# --------------------------------------------------------------------------- +# Run phases + + +def wait_for_terminal(attachment_ids, started_at, timeout=240): + deadline = time.monotonic() + timeout + timeline = [] + rows = [] + while time.monotonic() < deadline: + rows = job_rows(attachment_ids) + counts = {"queued": 0, "processing": 0, "ready": 0, "failed": 0} + for row in rows: + counts[row["status"]] = counts.get(row["status"], 0) + 1 + timeline.append( + { + "t": round(time.monotonic() - started_at, 3), + **counts, + } + ) + if len(rows) == len(attachment_ids) and all( + row["status"] in ("ready", "failed") for row in rows + ): + return rows, timeline + time.sleep(0.25) + raise Failure(f"jobs did not finish within {timeout}s; last={rows}") + + +def measure_run(base_url, actor, chat_id, profile, scenario, count, data, digest, + cpu_limit="1.0", mem_limit="2G"): + wait_drained() + recreate_worker( + { + "WORKER_CONCURRENCY": str(profile["concurrency"]), + "WORKER_PREFETCH": str(profile["prefetch"]), + "WORKER_CPU_LIMIT": str(cpu_limit), + "WORKER_MEM_LIMIT": str(mem_limit), + "WORKER_FAULT_MODE": "none", + "WORKER_CRASH_AFTER_EFFECT": "false", + "WORKER_CRASH_AFTER_CLAIM": "false", + } + ) + time.sleep(1.0) + container = worker_container() + + sampler = RssSampler(container) + sampler.start() + time.sleep(1.2) + baseline_samples = sampler.snapshot() + + submit_started = time.monotonic() + uploads = upload_many(base_url, actor, chat_id, data, count, f"track1-{profile['name']}-{scenario['name']}") + submit_seconds = time.monotonic() - submit_started + accepted = [item["attachment_id"] for item in uploads if item.get("attachment_id")] + rejected = [item for item in uploads if item["status"] != 202] + if rejected: + raise Failure(f"uploads were rejected during {profile['name']}/{scenario['name']}: {rejected[:3]}") + if len(accepted) != count: + raise Failure(f"only {len(accepted)}/{count} uploads were accepted") + + rows, timeline = wait_for_terminal(accepted, submit_started) + wall_seconds = time.monotonic() - submit_started + time.sleep(0.5) + samples = sampler.stop() + + failed = [row for row in rows if row["status"] != "ready"] + if failed: + raise Failure(f"jobs failed in {profile['name']}/{scenario['name']}: {failed[:2]}") + + queue_lag = [ + (row["processing_started_at"] - row["created_at"]) * 1000 + for row in rows + if row["processing_started_at"] is not None + ] + time_to_ready = [ + (row["finished_at"] - row["created_at"]) * 1000 + for row in rows + if row["finished_at"] is not None + ] + processing = [ + (row["finished_at"] - row["processing_started_at"]) * 1000 + for row in rows + if row["processing_started_at"] is not None and row["finished_at"] is not None + ] + upload_latency = [item["elapsed"] * 1000 for item in uploads] + + first_created = min(row["created_at"] for row in rows) + last_finished = max(row["finished_at"] for row in rows) + span = max(last_finished - first_created, 0.001) + throughput = len(rows) / span + + events = [] + for row in rows: + events.append((row["created_at"], 1)) + events.append((row["finished_at"], -1)) + backlog = 0 + max_backlog = 0 + for _, delta in sorted(events, key=lambda item: (item[0], -item[1])): + backlog += delta + max_backlog = max(max_backlog, backlog) + + # Actual parallel handlers: intersections of processing_started_at -> + # finished_at. These are bounded by WORKER_CONCURRENCY. + processing_events = [] + for row in rows: + if row["processing_started_at"] is None or row["finished_at"] is None: + continue + processing_events.append((row["processing_started_at"], 1)) + processing_events.append((row["finished_at"], -1)) + parallel = 0 + max_parallel = 0 + for _, delta in sorted(processing_events, key=lambda item: (item[0], -item[1])): + parallel += delta + max_parallel = max(max_parallel, parallel) + + baseline_kb = max(baseline_samples) if baseline_samples else None + sample_array = [value for value in samples if value > 0] + max_kb = max(sample_array) if sample_array else None + end_kb = sample_array[-1] if sample_array else None + + return { + "profile": profile["name"], + "scenario": scenario["name"], + "concurrency": profile["concurrency"], + "prefetch": profile["prefetch"], + "jobs": len(rows), + "image_width": scenario["width"], + "image_height": scenario["height"], + "image_bytes": len(data), + "image_sha256": digest, + "throughput_jobs_per_s": r(throughput, 3), + "pipeline_span_seconds": r(span, 3), + "submit_seconds": r(submit_seconds, 3), + "wall_seconds": r(wall_seconds, 3), + "upload_p50_ms": r(percentile(upload_latency, 50), 1), + "upload_p95_ms": r(percentile(upload_latency, 95), 1), + "queue_lag_p50_ms": r(percentile(queue_lag, 50), 1), + "queue_lag_p95_ms": r(percentile(queue_lag, 95), 1), + "queue_lag_max_ms": r(max(queue_lag) if queue_lag else None, 1), + "time_to_ready_p50_ms": r(percentile(time_to_ready, 50), 1), + "time_to_ready_p95_ms": r(percentile(time_to_ready, 95), 1), + "time_to_ready_max_ms": r(max(time_to_ready) if time_to_ready else None, 1), + "processing_p50_ms": r(percentile(processing, 50), 1), + "processing_p95_ms": r(percentile(processing, 95), 1), + "max_backlog_jobs": max_backlog, + "max_parallel_jobs": max_parallel, + "rss_baseline_mib": r(baseline_kb / 1024 if baseline_kb else None, 1), + "rss_peak_mib": r(max_kb / 1024 if max_kb else None, 1), + "rss_end_mib": r(end_kb / 1024 if end_kb else None, 1), + "rss_samples": len(sample_array), + "rss_delta_mib": r((max_kb - baseline_kb) / 1024 if max_kb and baseline_kb else None, 1), + "timeline": timeline, + } + + +# AGENT: backpressure is tested separately from the matrix: cap=3, worker down, +# sequential uploads until 503. The accepted jobs are the evidence that bounded +# acceptance did not lose already confirmed work. +def measure_backpressure(base_url, actor, chat_id, data, digest): + wait_drained() + recreate_api({"MAX_PENDING_JOBS": "3"}) + compose("stop", "worker") + wait_stopped("worker") + time.sleep(0.5) + + attempts = [] + accepted_ids = [] + for index in range(12): + item = upload_one(base_url, actor, chat_id, data, f"track1-backpressure-{index}.png") + item["index"] = index + attempts.append(item) + if item["status"] == 202: + accepted_ids.append(item["attachment_id"]) + if item["status"] == 503: + break + if not any(item["status"] == 503 for item in attempts): + raise Failure("MAX_PENDING_JOBS=3 did not produce a 503") + blocked = next(item for item in attempts if item["status"] == 503) + if blocked.get("retry_after") != "5": + raise Failure(f"503 did not carry Retry-After: 5 (got {blocked.get('retry_after')!r})") + + recovery_started = time.monotonic() + compose("start", "worker") + wait_healthy("worker") + rows, _ = wait_for_terminal(accepted_ids, recovery_started, timeout=180) + recovery_seconds = time.monotonic() - recovery_started + if any(row["status"] != "ready" for row in rows): + raise Failure("accepted jobs did not recover after worker start") + + wait_drained() + probe = upload_one(base_url, actor, chat_id, data, "track1-backpressure-probe.png") + if probe["status"] != 202: + raise Failure(f"probe upload after recovery returned {probe['status']}") + probe_ready_rows, _ = wait_for_terminal([probe["attachment_id"]], time.monotonic(), timeout=120) + if probe_ready_rows[0]["status"] != "ready": + raise Failure("probe upload did not become ready") + + return { + "cap": 3, + "attempts": [ + { + "index": item["index"], + "status": item["status"], + "elapsed_ms": r(item["elapsed"] * 1000, 1), + "retry_after": item.get("retry_after"), + "error_code": item.get("error_code"), + } + for item in attempts + ], + "accepted_before_503": len(accepted_ids), + "blocked_status": blocked["status"], + "blocked_retry_after": blocked.get("retry_after"), + "recovery_seconds": r(recovery_seconds, 3), + "recovered_jobs": len(rows), + "probe_status": probe["status"], + "probe_ready": True, + "image_bytes": len(data), + "image_sha256": digest, + } + + +# --------------------------------------------------------------------------- +# HTML dashboard (inline CSS + SVG only) + + +# AGENT: the dashboard is a single HTML file with inline CSS and SVG. No CDN, +# JS framework or external font is needed, so evidence stays readable offline. +COLORS = {"serial": "#64748b", "balanced": "#0284c7", "eager": "#f59e0b"} + + +def svg_chart(title, subtitle, categories, series, values, unit, digits=1, lower_better=False): + width, height = 760, 320 + left, right, top, bottom = 64, 20, 52, 62 + plot_w = width - left - right + plot_h = height - top - bottom + all_values = [v for group in values.values() for v in group if v is not None] + maximum = max(all_values) if all_values else 1.0 + maximum = maximum * 1.18 if maximum > 0 else 1.0 + parts = [ + f'', + f'{html.escape(title)}', + f'{html.escape(subtitle)}', + ] + for tick in range(5): + value = maximum * tick / 4 + y = top + plot_h - plot_h * tick / 4 + parts.append( + f'' + ) + parts.append( + f'' + f"{value:.0f}{html.escape(unit)}" + ) + group_width = plot_w / len(categories) + bar_width = group_width / (len(series) + 1.1) + for group_index, category in enumerate(categories): + base_x = left + group_index * group_width + bar_width * 0.35 + for series_index, name in enumerate(series): + value = values[name][group_index] + if value is None: + continue + bar_h = plot_h * value / maximum + x = base_x + series_index * bar_width + y = top + plot_h - bar_h + parts.append( + f'' + ) + label = f"{value:.{digits}f}{unit}" + parts.append( + f'{html.escape(label)}' + ) + parts.append( + f'' + f"{html.escape(category)}" + ) + # Legend + legend_x = left + for name in series: + parts.append( + f'' + ) + parts.append( + f'' + f"{html.escape(name)}" + ) + legend_x += 90 + hint = "lower is better" if lower_better else "higher is better" + parts.append( + f'{hint}' + ) + parts.append("") + return "\n".join(parts) + + +def build_html(evidence): + runs = evidence["runs"] + scenarios = [item["name"] for item in evidence["scenarios"]] + profiles = [item["name"] for item in evidence["profiles"]] + lookup = {(run["profile"], run["scenario"]): run for run in runs} + + def series_for(metric): + return { + profile: [ + (lookup.get((profile, scenario)) or {}).get(metric) + for scenario in scenarios + ] + for profile in profiles + } + + charts = [ + svg_chart( + "Throughput: готовые задания в секунду", + "Чем выше, тем быстрее pipeline обрабатывает очередь.", + scenarios, profiles, series_for("throughput_jobs_per_s"), " jobs/s", digits=2, + ), + svg_chart( + "p95 time-to-ready: от 202 до ready", + "95% заданий стали ready быстрее этого времени (меньше — лучше).", + scenarios, profiles, series_for("time_to_ready_p95_ms"), " ms", digits=0, lower_better=True, + ), + svg_chart( + "p95 queue lag: ожидание до старта обработки", + "Время между durable 202 и первым claim worker-а (меньше — лучше).", + scenarios, profiles, series_for("queue_lag_p95_ms"), " ms", digits=0, lower_better=True, + ), + svg_chart( + "Пиковый RSS worker", + "VmRSS процесса worker во время задания (меньше — экономнее).", + scenarios, profiles, series_for("rss_peak_mib"), " MiB", digits=0, lower_better=True, + ), + ] + + def fmt(value, digits=1, suffix=""): + return "—" if value is None else f"{value:.{digits}f}{suffix}" + + table_rows = [] + for run in runs: + table_rows.append( + "" + f"{html.escape(run['profile'])}" + f"{html.escape(run['scenario'])}" + f"{run['jobs']}" + f"{fmt(run['throughput_jobs_per_s'], 2)}" + f"{fmt(run['queue_lag_p95_ms'], 0)}" + f"{fmt(run['time_to_ready_p95_ms'], 0)}" + f"{fmt(run['processing_p95_ms'], 0)}" + f"{fmt(run['rss_peak_mib'], 0)}" + f"{run['max_parallel_jobs']}" + f"{run['max_backlog_jobs']}" + "" + ) + + # Beginner-friendly observations computed from the actual numbers. + observations = [] + for scenario in scenarios: + serial = lookup.get(("serial", scenario)) + eager = lookup.get(("eager", scenario)) + if serial and eager and serial["throughput_jobs_per_s"]: + speedup = eager["throughput_jobs_per_s"] / serial["throughput_jobs_per_s"] + if speedup >= 1.05: + comparison = f"eager быстрее serial в {speedup:.1f}× по throughput" + elif speedup <= 0.95: + comparison = f"eager медленнее serial ({speedup:.2f}×) по throughput" + else: + comparison = "throughput eager и serial практически одинаков" + observations.append(f"{scenario}: {comparison}.") + if serial["rss_peak_mib"] and eager["rss_peak_mib"]: + ratio = eager["rss_peak_mib"] / serial["rss_peak_mib"] + observations.append( + f"{scenario}: пиковый RSS eager относится к serial как {ratio:.1f}×." + ) + balanced = lookup.get(("balanced", "small_many")) + huge = lookup.get(("balanced", "huge_few")) + if balanced and huge and balanced["time_to_ready_p95_ms"] and huge["time_to_ready_p95_ms"]: + observations.append( + "При balanced p95 time-to-ready для huge_few в " + f"{huge['time_to_ready_p95_ms'] / balanced['time_to_ready_p95_ms']:.1f}× " + "больше, чем для small_many: размер изображения меняет стоимость задания, " + "а не только число заданий." + ) + huge_serial = lookup.get(("serial", "huge_few")) + huge_eager = lookup.get(("eager", "huge_few")) + if huge_serial and huge_eager and huge_serial["throughput_jobs_per_s"]: + speedup = huge_eager["throughput_jobs_per_s"] / huge_serial["throughput_jobs_per_s"] + if speedup < 1.2: + observations.append( + "На huge_few eager почти не поднял throughput " + f"({speedup:.2f}×): worker ограничен " + f"{html.escape(str(evidence['environment']['worker_cpu_limit']))} CPU, " + "а декодирование и масштабирование упираются в CPU. Concurrency " + "снижает queue lag, но при этом растёт RSS; это и есть компромисс " + "latency/ресурсы, а не бесплатное ускорение." + ) + else: + observations.append( + f"На huge_few eager поднял throughput в {speedup:.2f}× относительно " + "serial, потому что несколько тяжёлых заданий перекрываются." + ) + observations.append( + "serial ограничивает одновременно обрабатываемые задания одним, поэтому " + "queue lag растёт, но RSS остаётся минимальным. eager держит больше " + "заданий в работе и в prefetch-окне брокера." + ) + observations.append( + "Backpressure: при MAX_PENDING_JOBS=3 API принял " + f"{evidence['backpressure']['accepted_before_503']} задания, затем вернул 503 " + f"с Retry-After={evidence['backpressure']['blocked_retry_after']}; после старта " + f"worker очередь восстановилась за {evidence['backpressure']['recovery_seconds']:.1f} с." + ) + + backpressure_rows = "".join( + "" + f"{item['index'] + 1}" + f"{item['status']}" + f"{fmt(item['elapsed_ms'], 1)}" + f"{html.escape(str(item['retry_after'] or '—'))}" + f"{html.escape(str(item['error_code'] or '—'))}" + "" + for item in evidence["backpressure"]["attempts"] + ) + + observations_html = "".join(f"
  • {html.escape(text)}
  • " for text in observations) + chart_html = "\n".join(f'
    {chart}
    ' for chart in charts) + + return f""" + + + + +Lab5 track 1 — очередь под перегрузкой + + + +
    +

    Lab5, трек 1: очередь под перегрузкой

    +

    Сгенерировано {html.escape(evidence['generated_at'])}. Стенд: {html.escape(evidence['environment']['base_url'])}, +{evidence['environment']['cpu_count']} CPU, Docker {html.escape(evidence['environment']['docker_version'])}.

    + +
    +Гипотеза. Маленькие изображения, которые почти не требуют +масштабирования, должны давать высокий throughput и низкий RSS на задание. +Большие изображения требуют декодирования и downscale, поэтому каждое задание +дороже: при том же числе worker-ов очередь растёт, p95 time-to-ready +увеличивается, а пиковый RSS зависит от concurrency сильнее, чем у маленьких +файлов. Одновременно проверяем компромисс: eager уменьшает ожидание, +но повышает потребление памяти; serial экономит память, но ограничивает +throughput. +
    + +
    +

    Профили и сценарии

    + + + +{''.join(f"" for p in evidence['profiles'])} + +
    ПрофильConcurrencyPrefetchСмысл
    {html.escape(p['name'])}{p['concurrency']}{p['prefetch']}{html.escape(p['note'])}
    +

    Сценарии: +{'; '.join(f"{html.escape(s['name'])} — {s['width']}×{s['height']}, {s['count']} jobs, {s['description']}" for s in evidence['scenarios'])}.

    +
    + +
    +

    Ключевые числа

    +
    +
    {len(runs)}
    прогонов матрицы
    +
    {max(run['rss_peak_mib'] or 0 for run in runs):.0f} MiB
    максимальный RSS worker
    +
    {evidence['backpressure']['accepted_before_503']}
    принято до первого 503
    +
    {evidence['backpressure']['recovery_seconds']:.1f} s
    восстановление после backpressure
    +
    +
    + +{chart_html} + +
    +

    Полная матрица

    + + +{''.join(table_rows)} +
    ПрофильСценарийJobsJobs/sQueue lag p95, msReady p95, msProcessing p95, msRSS peak, MiBMax parallelMax backlog
    +

    Queue lag = processing_started_at - created_at. +Time-to-ready = finished_at - created_at. Processing = finished_at - processing_started_at. +RSS — VmRSS процесса worker, сэмплирование примерно каждые 100 мс. +Max parallel — максимум одновременно обрабатываемых заданий; max backlog — +максимум принятых, но ещё не готовых заданий.

    +
    + +
    +

    Backpressure и восстановление

    +

    Перед этим прогоном API пересоздан с MAX_PENDING_JOBS=3, worker +остановлен. Последовательные upload-ы показывают, где заканчивается принятая +работа. Затем worker запускается, принятые задания должны дойти до ready.

    + + +{backpressure_rows} +
    #HTTPLatency, msRetry-AfterError code
    +

    Recovery: {evidence['backpressure']['recovered_jobs']} принятых заданий +стали ready за {evidence['backpressure']['recovery_seconds']:.2f} с после старта worker. +Финальный probe upload: HTTP {evidence['backpressure']['probe_status']} → ready.

    +
    + +
    +

    Что это значит

    +
      {observations_html}
    +
    + +
    +

    Ограничения

    +
      +
    • Один worker, один RabbitMQ и один PostgreSQL: это сравнение настроек, а не +HA-кластер. Числа зависят от CPU/диска конкретной машины.
    • +
    • PNG-фикстуры синтетические и хорошо сжимаются; доминирует стоимость декодирования +и масштабирования, а не передача по сети. Размер в байтах указан в JSON.
    • +
    • RSS сэмплируется из /proc/1/status; короткий прогон не заменяет +длительный memory profile и p99.
    • +
    • Backpressure cap приблизительный: каждая API-реплика считает pending-задания +по общей БД, но проверка и вставка не являются одной транзакцией.
    • +
    +
    +
    + + +""" + + +def write_markdown(evidence, path): + lines = [ + "# Lab5 track 1 — queue under load", + "", + f"Generated: `{evidence['generated_at']}`", + "", + "Synthetic images; no tokens or credentials are stored.", + "", + "| Profile | Scenario | Jobs | Jobs/s | Queue lag p95, ms | Ready p95, ms | RSS peak, MiB |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: |", + ] + for run in evidence["runs"]: + lines.append( + f"| {run['profile']} | {run['scenario']} | {run['jobs']} | " + f"{run['throughput_jobs_per_s']:.2f} | {run['queue_lag_p95_ms']:.0f} | " + f"{run['time_to_ready_p95_ms']:.0f} | {run['rss_peak_mib']:.0f} |" + ) + bp = evidence["backpressure"] + lines.extend( + [ + "", + f"Backpressure: cap={bp['cap']}, accepted={bp['accepted_before_503']}, " + f"first 503 Retry-After={bp['blocked_retry_after']}, " + f"recovery={bp['recovery_seconds']:.2f}s, probe HTTP {bp['probe_status']}.", + "", + ] + ) + path.write_text("\n".join(lines)) + + +# --------------------------------------------------------------------------- +# Main + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) + parser.add_argument("--small-jobs", type=int, default=30) + parser.add_argument("--huge-jobs", type=int, default=12) + parser.add_argument( + "--worker-cpu-limit", + default=os.environ.get("TRACK1_WORKER_CPU_LIMIT", "1.0"), + help="Compose CPU quota applied to every experimental worker (default 1.0)", + ) + parser.add_argument( + "--worker-mem-limit", + default=os.environ.get("TRACK1_WORKER_MEM_LIMIT", "2G"), + help="Compose memory limit applied to every experimental worker (default 2G)", + ) + parser.add_argument("--out", default=str(ROOT / "evidence" / "track1.json")) + parser.add_argument("--html-out", default=str(ROOT / "evidence" / "track1.html")) + parser.add_argument("--md-out", default=str(ROOT / "evidence" / "track1.md")) + parser.add_argument( + "--skip-backpressure", + action="store_true", + help="only run the profile x scenario matrix", + ) + args = parser.parse_args() + + base_url = args.base_url.rstrip("/") + out = Path(args.out) + html_out = Path(args.html_out) + md_out = Path(args.md_out) + out.parent.mkdir(parents=True, exist_ok=True) + html_out.parent.mkdir(parents=True, exist_ok=True) + + print(f"== bootstrap {base_url}", flush=True) + client = Client(base_url, timeout=60) + alice, _bob, _eve, chat = client.bootstrap() + + scenarios = [] + for item in SCENARIOS: + count = args.small_jobs if item["name"] == "small_many" else args.huge_jobs + data = make_png(item["width"], item["height"]) + scenarios.append({**item, "count": count, "bytes": len(data)}) + + try: + wait_drained() + runs = [] + for profile in PROFILES: + for scenario in scenarios: + data = make_png(scenario["width"], scenario["height"]) + if len(data) != scenario["bytes"]: + raise Failure("fixture generation is not deterministic") + digest = hashlib.sha256(data).hexdigest() + print( + f"== run {profile['name']}/{scenario['name']} " + f"({scenario['count']} jobs, {scenario['width']}x{scenario['height']})", + flush=True, + ) + result = measure_run( + base_url, + alice, + chat["id"], + profile, + scenario, + scenario["count"], + data, + digest, + cpu_limit=args.worker_cpu_limit, + mem_limit=args.worker_mem_limit, + ) + runs.append(result) + print( + f" throughput={result['throughput_jobs_per_s']} jobs/s " + f"queue_p95={result['queue_lag_p95_ms']} ms " + f"ready_p95={result['time_to_ready_p95_ms']} ms " + f"rss_peak={result['rss_peak_mib']} MiB", + flush=True, + ) + + backpressure = None + if not args.skip_backpressure: + print("== backpressure (cap=3, worker stopped)", flush=True) + small = scenarios[0] + backpressure = measure_backpressure( + base_url, + alice, + chat["id"], + make_png(small["width"], small["height"]), + hashlib.sha256(make_png(small["width"], small["height"])).hexdigest(), + ) + print( + f" accepted={backpressure['accepted_before_503']} " + f"503_retry_after={backpressure['blocked_retry_after']} " + f"recovery={backpressure['recovery_seconds']}s", + flush=True, + ) + finally: + # Restore the stand even if a measurement raised. + try: + restore_worker() + except Exception as exc: # noqa: BLE001 - best effort in cleanup + print(f"warning: worker restore failed: {exc}", flush=True) + try: + restore_api() + except Exception as exc: # noqa: BLE001 - best effort in cleanup + print(f"warning: API restore failed: {exc}", flush=True) + + if backpressure is None: + backpressure = { + "cap": 3, + "attempts": [], + "accepted_before_503": 0, + "blocked_status": None, + "blocked_retry_after": None, + "recovery_seconds": 0.0, + "recovered_jobs": 0, + "probe_status": None, + "probe_ready": False, + } + + docker_version = compose_capture("version", "--short").strip() + evidence = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "environment": { + "base_url": base_url, + "host": platform.node(), + "platform": platform.platform(), + "cpu_count": os.cpu_count(), + "python": platform.python_version(), + "docker_version": docker_version, + "compose_project": "mtusi2026lab", + "worker_container": worker_container(), + "worker_cpu_limit": args.worker_cpu_limit, + "worker_mem_limit": args.worker_mem_limit, + "worker_cpu_limit_default": "1.0", + "worker_mem_limit_default": "1G", + "note": ( + "Experimental workers use a larger memory limit than .env so the " + "eager huge-image run can hold decoded images without OOM; CPU is " + "still limited so throughput is not a host-CPU artifact." + ), + }, + "hypothesis": ( + "Small images that fit the variants without large scaling should be cheaper " + "per job; huge images force decode+downscale, so each job costs more memory " + "and time. More concurrency should raise throughput and RSS but reduce queue lag." + ), + "profiles": PROFILES, + "scenarios": scenarios, + "runs": runs, + "backpressure": backpressure, + } + + out.write_text(json.dumps(evidence, indent=2) + "\n") + html_out.write_text(build_html(evidence)) + write_markdown(evidence, md_out) + print(json.dumps({"ok": True, "json": str(out), "html": str(html_out), "md": str(md_out)})) + + +if __name__ == "__main__": + try: + main() + except (Failure, RuntimeError, subprocess.TimeoutExpired) as exc: + print("FAIL:", exc) + raise SystemExit(1)