From ef1f470659610f8ee0dc20bc3315589ba18154ca Mon Sep 17 00:00:00 2001 From: Sergey Lyakhonov Date: Fri, 20 Nov 2026 07:05:16 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9F=D0=BE=D0=B4=D1=82=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D0=B4=D0=B8=D1=82=D1=8C=20=D1=82=D1=80=D0=B5=D0=BA=202=20?= =?UTF-8?q?=D1=80=D0=B5=D0=B0=D0=BB=D1=8C=D0=BD=D1=8B=D0=BC=D0=B8=20=D1=81?= =?UTF-8?q?=D0=B1=D0=BE=D1=8F=D0=BC=D0=B8,=20=D0=BD=D0=B0=D0=B3=D1=80?= =?UTF-8?q?=D1=83=D0=B7=D0=BA=D0=BE=D0=B9=20=D0=B8=20=D0=BE=D1=82=D1=87?= =?UTF-8?q?=D1=91=D1=82=D0=BE=D0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 212 +- evidence/audit.json | 13 + evidence/environment.json | 263 ++ evidence/failover-each.json | 336 ++ evidence/failover-targeted-run.txt | 1 + evidence/failover.json | 176 + evidence/fault-environment.json | 240 ++ evidence/faults.png | Bin 0 -> 61476 bytes evidence/final-state.json | 35 + evidence/isolation.txt | 2 + evidence/lab4-baseline.json | 126 + evidence/load-messages-resources.json | 775 ++++ evidence/load-messages-run.txt | 17 + evidence/load-messages.json | 17 + evidence/load-pipeline.json | 3520 +++++++++++++++++ evidence/load-run.txt | 6 + evidence/load.png | Bin 0 -> 47604 bytes evidence/measurements.csv | 7 + evidence/normal-start-retry.txt | 37 + evidence/normal-start.txt | 80 + evidence/persistence.json | 176 + evidence/pipeline-initial-run.txt | 56 + evidence/pipeline-resumed-run.txt | 8 + evidence/pipeline-run.txt | 56 + evidence/pipeline.json | 5145 +++++++++++++++++++++++++ evidence/resilience-initial-run.txt | 37 + evidence/resilience-run.txt | 37 + evidence/smoke.txt | 1 + evidence/structure.txt | 1 + evidence/unit.txt | 5 + evidence/validation.txt | 1 + requirements-research.txt | 2 + scripts/audit.py | 2 +- scripts/environment.py | 2 + scripts/final_state.py | 36 + scripts/reproduce.py | 13 +- scripts/summarize.py | 43 + scripts/wait_ready.py | 44 + student-tests/load_messages.py | 22 + student-tests/load_pipeline.py | 2 +- student-tests/pipeline.py | 1 + student-tests/probe.py | 11 + student-tests/resilience_driver.py | 4 +- 43 files changed, 11538 insertions(+), 30 deletions(-) create mode 100644 evidence/audit.json create mode 100644 evidence/environment.json create mode 100644 evidence/failover-each.json create mode 100644 evidence/failover-targeted-run.txt create mode 100644 evidence/failover.json create mode 100644 evidence/fault-environment.json create mode 100644 evidence/faults.png create mode 100644 evidence/final-state.json create mode 100644 evidence/isolation.txt create mode 100644 evidence/lab4-baseline.json create mode 100644 evidence/load-messages-resources.json create mode 100644 evidence/load-messages-run.txt create mode 100644 evidence/load-messages.json create mode 100644 evidence/load-pipeline.json create mode 100644 evidence/load-run.txt create mode 100644 evidence/load.png create mode 100644 evidence/measurements.csv create mode 100644 evidence/normal-start-retry.txt create mode 100644 evidence/normal-start.txt create mode 100644 evidence/persistence.json create mode 100644 evidence/pipeline-initial-run.txt create mode 100644 evidence/pipeline-resumed-run.txt create mode 100644 evidence/pipeline-run.txt create mode 100644 evidence/pipeline.json create mode 100644 evidence/resilience-initial-run.txt create mode 100644 evidence/resilience-run.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/structure.txt create mode 100644 evidence/unit.txt create mode 100644 evidence/validation.txt create mode 100644 requirements-research.txt create mode 100644 scripts/final_state.py create mode 100644 scripts/summarize.py create mode 100644 scripts/wait_ready.py create mode 100644 student-tests/load_messages.py diff --git a/REPORT.md b/REPORT.md index e98f75e9..9fb356b5 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,207 @@ -# Отчёт — лабораторная 5 +# Лабораторная №5 — очереди и асинхронная обработка изображений -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Ляхонов Сергей, БВТ2301. 20 ноября 2026 года.** Цель: **5**, выбран **трек 2 — доказанная идемпотентность versioned pipeline**. Ветка `solution/BVT2301-lyakhonov`. Проверенная ревизия — в [reproduction.json](evidence/reproduction.json), авторство — в [audit.json](evidence/audit.json). ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Desktop с Linux-контейнерами и Compose v2, Python 3.11+ для собственных генераторов (публичным тестам достаточно 3.9+). Проверено: Windows, Python хоста 3.13.7, Python контейнера 3.11.10, Engine 28.5.1, Compose 2.40.3-desktop.1, Intel i5-1340P, 16 логических CPU/16 GB RAM. Docker VM: 16 CPU/8 126 664 704 bytes RAM. ID образов, контейнеров, лимиты и параметры эксперимента: [environment.json](evidence/environment.json). -## Архитектура и решения +PowerShell из корня №5: -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```powershell +$env:PYTHONUTF8='1' +python scripts/bootstrap.py +docker compose -p lyakhonov-lab5 config --quiet +docker compose -p lyakhonov-lab5 build api s3 +docker compose -p lyakhonov-lab5 up -d +python scripts/wait_ready.py +python scripts/check_contract.py +python tests/smoke.py --base-url http://127.0.0.1:8085 +python -m venv .venv +.venv/Scripts/python -m pip install -r requirements-dev.txt +.venv/Scripts/python scripts/validate_openapi.py +docker compose -p lyakhonov-lab5 run --rm --no-deps -v "${PWD}:/suite:ro" -w /suite api python -m unittest discover -s student-tests -p 'test_*.py' +python student-tests/resilience_driver.py +``` -## Доказательства критериев +`make check validate test PYTHON=python BASE_URL=http://127.0.0.1:8085` эквивалентен прямым командам при наличии make и validator. UTF-8 mode обязателен на этом Windows: без него исходный публичный checker получает `UnicodeDecodeError` при чтении UTF-8 JSON как cp1251. Checker не изменён. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +Bootstrap создаёт **только отсутствующие** `.env` и `secrets/`, не перезаписывает настройки. Для чистого экспорта генерируются новые секреты. `migrate` применяет 001–006 в транзакции под advisory lock до запуска API. Повторные migrate, bucket-init и объявления очереди безопасны. Секреты передаются файлами `/run/secrets`, исключены из Git и build context. + +Рабочая конфигурация — `compose.yaml` и `lab5.env.example`. Исходный `.env.example` задания сохранён: сессии фактически в PostgreSQL, кэш через внутренний `CACHE_URL`, storage credentials отображены на `S3_ACCESS_KEY_FILE/S3_SECRET_KEY_FILE`, bucket — `attachments`. Лимиты bytes/pixels и 24h TTL идемпотентности фиксированы контрактом в коде. `WORKER_CONCURRENCY` — число worker-контейнеров, каждый имеет prefetch=1. + +Единственный опубликованный порт — `127.0.0.1:8085` HAProxy. При занятости задайте свободный `LAB5_PORT` и соответствующий URL тестов. Внутренние API, DB, broker, Redis, S3 не опубликованы. Тома/сеть принадлежат отдельному проекту `lyakhonov-lab5`, образы — `lyakhonov-messenger:lab5`, `lyakhonov-storage:lab5`. + +MinIO/mc собираются из официальных исходников: MinIO commit `07c3a429bfed433e49018cb0f78a52145d4bedeb`, mc `RELEASE.2025-04-16T18-13-26Z`. Архивы проверяются `ADD --checksum`, Go/Debian закреплены digest, Go modules — go.sum. Лицензии включены в образ. Сборка действительно прошла с использованием build cache; повторной недоступности официального MinIO-образа не утверждаю. Без кэша нужен доступ к Docker Hub/GitHub/Python package index/Go proxy. + +Остановка с сохранением данных: `docker compose -p lyakhonov-lab5 down`. Только при намеренном удалении своих учебных данных: `docker compose -p lyakhonov-lab5 down --volumes`. В persistence/failover volumes не удаляются. POSIX: python3, `.venv/bin/python`, executable `scripts/restart.sh`/`stop-one.sh`; Windows использует `.cmd` hooks. + +## Архитектура и перенос + +```mermaid +flowchart LR + C[Клиент] --> LB[HAProxy :8085] + LB --> A[2 FastAPI API] + A --> DB[(PostgreSQL: metadata, sessions, messages, jobs/outbox)] + A --> S[(MinIO: staging и final)] + D[Dispatcher: publish confirm, recovery] --> DB + D --> Q[(RabbitMQ durable queue)] + Q --> W[2 worker, prefetch 1] + W --> S + W --> DB + SC[Staging cleaner] --> DB + SC --> S +``` + +Из собственной №4 перенесены мессенджер, Argon2id/серверные сессии, keyset-пагинация, транзакционная идемпотентность сообщений, Redis-кэш профилей, приватный S3, HAProxy и рецепты сборки. История №4 не копировалась. Материалы №5, contracts, публичные tests/fixtures/examples сохранены. Существующие коммиты временного клона №5 `5894da9`, `a0ec26d`, `e376ede` перенесены без переписывания. Первоначальные незакоммиченные файлы сохранены отдельной копией вне проекта до переключения ветки. + +Изменения: upload `201 ready` → `202 queued`, устойчивый staging, jobs/outbox, отдельные broker/dispatcher/worker/cleaner, lease/retry/replay, versioned manifests. Reconciler исследования №4 не является обязательной частью №5; результаты №4 не выдаются за новые. №4 использована только для чтения: [fingerprint](evidence/lab4-baseline.json), [аудит](evidence/audit.json). + +| Роль/сервис | Версия | Готовность и данные | CPU/RAM | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| `api` ×2 | Python 3.11.10, FastAPI 0.115.0 | readiness PostgreSQL+S3; без локального durable state | 0.5/640 MiB каждый | +| `db` | PostgreSQL 16.6-alpine | pg_isready, postgres_data | 1/512 MiB | +| `s3` | закреплённый MinIO commit | bucket-init, s3_data | 1/768 MiB | +| `broker` | RabbitMQ 4.1.0-alpine | diagnostics ping, broker_data | 0.75/512 MiB | +| `worker` ×2 | приложение, Pillow 12.0.0, pika 1.3.2 | отдельный процесс, manual ack, restart unless-stopped | 0.75/768 MiB каждый | +| `dispatcher` | приложение | publisher confirm, recovery deadlines | 0.5/256 MiB | +| `staging-cleaner` | приложение | цикл 30 s, locks и реестр | 0.25/256 MiB | +| `redis` | Redis 7.4.1-alpine | необязательный восстанавливаемый кэш | 0.25/128 MiB | +| `lb` | HAProxy 3.0.6-alpine | HTTP readiness backend, round-robin | 0.5/128 MiB | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Сумма runtime-лимитов: **6.75 CPU/5376 MiB**, плюс одноразовые migrate/bucket-init и генератор. Это лимиты cgroup, не фактическое потребление. Полные зависимости закреплены в `requirements.lock.txt`. Graceful shutdown API — 8 s внутри 12 s Docker grace, worker — до 95 s; crash-тесты намеренно обходят graceful shutdown. -## Эксперименты и отказы +## Границы надёжности и инварианты -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +1. До 202 API принимает и проверяет **все** байты, пишет `staging/{attachment_id}` в приватный persistent S3; затем одной PostgreSQL-транзакцией создаёт attachment и job/outbox. Ответ — queued snapshot после COMMIT. Разрыв staging/DB может оставить неподтверждённый orphan; он не считается принятой задачей. +2. Dispatcher резервирует committed job с durable deadline 15 s, публикует ID/version/attempt/time, ждёт publisher confirm и отмечает публикацию. Смерть до publish или до отметки восстанавливается deadline и допускает повторную доставку. Байтов/credentials/произвольных URL в broker нет. +3. Логическая работа уникальна по `(attachment_id,pipeline_version)`, эффект по `(attachment_id,pipeline_version,variant)`. Claim блокирует attachment, затем job, сверяет envelope/attempt/target, выдаёт fencing token и lease. Дубль ready/processing или устаревшая версия не создаёт новую попытку. Durable lease восстанавливает работу владельца даже если другой дубль уже ack-нут. +4. Worker проверяет SHA-256/size staging, выполняет Pillow в отдельном subprocess, пишет immutable original и versioned variants с `If-None-Match: *`. Существующий key должен совпадать по metadata SHA-256/size/MIME. Manifest и ready публикуются одной DB-транзакцией только при актуальных token, lease и target version; затем ack. +5. Upgrade атомарно повышает target version и помечает старые активные jobs superseded. Старый живой worker может записать только keys своей версии и не может заменить новый manifest. Уже опубликованная v1 остаётся доступной до COMMIT v2; неуспешный upgrade виден в failed-реестре, прежний ready остаётся доступным. +6. Инварианты тестов: принятые задачи завершаются после исследованных сбоев; 3 manifest-записи на опубликованную версию; original/варианты скачиваются и совпадают с SHA-256; дубль не размножает keys; old job и old live worker не меняют published version/bytes v2. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +Это **at-least-once**, не exactly-once доставка. Повторяются доставка, resize и PUT; единственен логический опубликованный эффект. Publisher confirm — приём брокером, consumer ack — обработка доставки. Общей транзакции PostgreSQL/S3/RabbitMQ нет. + +Версии `src/pipeline_versions.py`: v1 JPEG quality=85/WebP=82, v2 JPEG=90/WebP=88. Байты вариантов действительно отличаются; original неизменен. Thumbnail вписан в 256×256 без увеличения, floor/minimum 1 px; crop — центрированный JPEG 128×128; optimized — WebP в 1280×1280 без увеличения, method=4. EXIF orientation применяется, метаданные удаляются, alpha компонуется на белый. Уменьшение каждого маленького файла не обещается. + +Ограничения: ≤10 MiB, ≤40 MP до полного decode, один upload/API-процесс, приём ≤20 s; worker subprocess timeout 60 s и worker cgroup 768 MiB. Lease по умолчанию 90 s, минимум timeout+5 s; prefetch=1. Для коротких fault-экспериментов использованы timeout=5 s/lease=10 s: [fault-environment.json](evidence/fault-environment.json). Их время восстановления нельзя объявлять временем production-конфигурации. + +До 5 управляемых попыток. Backoff `min(30,2^(attempt-1))*uniform(0.75,1.25)` s. Invalid image/integrity/object conflict постоянны; S3/timeout временны; исчерпание — `retry_exhausted`. Failed-реестр хранится в PostgreSQL, отдельная DLQ не нужна. Replay проверяет доступность source, записывает audit и выдаёт новый ограниченный бюджет. Ready replay не выполняет. + +```powershell +docker compose -p lyakhonov-lab5 exec -T --index 1 worker python -m src.operator failed +docker compose -p lyakhonov-lab5 exec -T --index 1 worker python -m src.operator replay ATTACHMENT_UUID +docker compose -p lyakhonov-lab5 exec -T --index 1 worker python -m src.operator upgrade ATTACHMENT_UUID --version 2 +``` + +Это локальные операторские расширения, обязательные HTTP endpoint/схемы не изменены. Staging ready очищается после безопасного manifest; failed/orphan retention 86400 s, для failed отсчёт от settled_at. Queued/processing не удаляются. Cleanup и upload/replay/upgrade согласованы advisory locks. Финальные original и старые versioned variants автоматически не удаляются. + +## Проверки и доказательства + +| Критерий | Команда/сценарий | Наблюдение | Доказательство | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Материалы/контракт | checker, validator, audit | структура, OpenAPI 3.1/JSON Schema, исходные материалы не изменены | [structure](evidence/structure.txt), [validation](evidence/validation.txt), [audit](evidence/audit.json) | +| Минимум №4 с новым upload | public smoke | auth, messages, pagination, ACL, идемпотентность, 202→ready, original/thumbnail | [smoke](evidence/smoke.txt) | +| Durable handoff/broker/worker | worker stop → upload/message → API+broker restart → worker start | тот же attachment ID доходит до ready | [pipeline](evidence/pipeline.json), [environment](evidence/environment.json) | +| Состояния и варианты | unit/negative/durable suite | queued/processing/ready/failed, реальные JPEG/WebP и checksums | [unit](evidence/unit.txt), [pipeline](evidence/pipeline.json) | +| Bytes/pixels/MIME/name/ACL | negative suite | 413/415/400/404, 24 конкурентных POST одного ключа дают 1 message ID | [pipeline](evidence/pipeline.json) | +| Outbox COMMIT→publish | before_publish ×3 | смерть dispatcher, автоматическое ready без ручной новой job | [pipeline](evidence/pipeline.json) | +| Publish→confirm bookkeeping | after_publish ×3 | смерть до DB-отметки, повтор безопасен | [pipeline](evidence/pipeline.json) | +| Lease/side effect/ack | before_effect, after_original, after_variants, after_commit ×3 | os._exit(91), restart; те же bytes и 3 manifest-записи | [pipeline](evidence/pipeline.json) | +| Retry/failed/replay | remove/corrupt staging | постоянный отказ за 1 попытку, временный — 5, replay с audit до ready | [pipeline](evidence/pipeline.json) | +| Очистка staging | cleanup после ready → upgrade | v2 построена из immutable original после очистки | [pipeline](evidence/pipeline.json) | +| Трек 2: duplicate/reorder/stale | version suite ×3 | 52 явно опубликованных сообщения/повтор, старая paused попытка освобождена после v2, новый результат сохранён | [pipeline](evidence/pipeline.json) | +| Persistence stores | resilience_driver | публичный persistence и реальные StartedAt API/DB/S3/broker/Redis | [persistence](evidence/persistence.json) | +| Failover API | resilience_driver | публичный failover + отдельная остановка каждой API, сохранность подтверждений | [public](evidence/failover.json), [каждая API](evidence/failover-each.json) | +| Нагрузка полного pipeline | load_pipeline | completed images, ready latency, ошибки, CPU/memory | [load](evidence/load-pipeline.json) | +| Чистое воспроизведение | scripts/reproduce.py | Git export, новые secrets/images/project/volumes/port, build+smoke, cleanup своих временных ресурсов | [reproduction](evidence/reproduction.json) | -## Ограничения и следующие шаги +Fault hooks выключены по умолчанию, включаются только окружением `LAB5_FAULT_INJECTION=1`, не через HTTP. `experiment_faults` фиксирует arm/hit/process_id. Crash — реальная смерть процесса без finally/ack. Pause удерживает старую попытку до публикации новой. Каждый arm одноразовый. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Одноразовый `inspector` профиля `tools` нужен только операторскому тесту списка S3-объектов; он не запускается обычным `up` и не публикует порты. Root credentials монтируются только в этот одноразовый процесс и bucket-init. Приложению список final-prefix не разрешён: первый запуск тестового probe получил `AccessDenied`, что сохранено в `pipeline-initial-run.txt`. Инспектор исправлен без расширения policy приложения; crash-результаты сохранены, version/error suite продолжен с `--resume`. + +```powershell +$env:LAB5_FAULT_INJECTION='1' +$env:JOB_TIMEOUT_SECONDS='5' +$env:JOB_LEASE_SECONDS='10' +docker compose -p lyakhonov-lab5 up -d +python scripts/wait_ready.py +python student-tests/pipeline.py --repeats 3 +Remove-Item Env:LAB5_FAULT_INJECTION,Env:JOB_TIMEOUT_SECONDS,Env:JOB_LEASE_SECONDS +docker compose -p lyakhonov-lab5 up -d +python scripts/wait_ready.py +python student-tests/load_pipeline.py --duration 20 --concurrency 2 --repeats 3 +python tests/load.py --base-url http://127.0.0.1:8085 --duration 30 --concurrency 4 --output evidence/load-messages.json +python scripts/environment.py +python scripts/reproduce.py +python scripts/audit.py +``` + +## Методика и результаты + +Гипотеза: fencing публикации manifest и immutable versioned S3 keys сохраняют принятые задачи и результат новой версии при дублях, reorder и процессных crash/restart в исследованной модели. + +Fault suite использует fixture 320×200, 111817 bytes. Шесть окон crash ×3 повтора. График показывает **полную длительность сценария** (stop, upload, start, ready, downloads), не чистую latency восстановления. Сохранены ID/StartedAt и состояния до/после, checksums всех четырёх скачанных результатов. + +![Реальные crash-сценарии](evidence/faults.png) + +Нагрузка closed-loop: 2 клиента, 20 s запуска операций, один завершённый warmup/профиль/повтор, 3 повтора. Профили: публичный PNG 320×200 и детерминированный RGB PNG 1024×768 с псевдослучайными пикселями. Клиент ждёт **ready и три варианта**, затем начинает следующий upload. Знаменатель successful RPS — фактическая длительность с завершением последней операции. Ошибки сохраняются, прозрачных retry upload нет. Длительность включает передачу bytes, очередь, обработку и polling. + + + +| Профиль | Bytes/image | Завершено за 3 повтора | Успешных image/s (min–max) | p50 ready, s | p95 ready, s | p99 ready, s | Ошибки | +| --- | --- | --- | --- | --- | --- | --- | --- | +| 1024x768 | 2360847 | 117 | 1.569–2.188 | 0.790–1.067 | 1.140–1.230 | 1.161–1.322 | 1 | +| 320x200 | 111817 | 224 | 3.549–3.982 | 0.401–0.439 | 0.727–0.769 | 0.744–0.820 | 0 | + +В собранных snapshots максимальная сумма CPU контейнеров: **218.3%** (100% = один CPU); сумма cgroup memory: **1047.5 MiB**. Это максимум редких снимков, не непрерывно измеренный пик. + +| Точка crash | Повторов | Полная длительность сценария, s (min–max) | +| --- | --- | --- | +| before_publish | 3 | 28.940–32.205 | +| after_publish | 3 | 14.478–16.650 | +| before_effect | 3 | 25.678–28.623 | +| after_original | 3 | 25.642–26.509 | +| after_variants | 3 | 25.458–27.131 | +| after_commit | 3 | 14.989–17.081 | + +Во всех сохранённых crash/version/replay-сценариях проверки инвариантов прошли. Увеличение входного изображения снижает throughput и повышает время до ready в измеренном профиле. Повторы показывают разброс, а не статистически надёжную оценку хвостов: выборка и длительность малы. + + + +![Throughput и p95 полного pipeline](evidence/load.png) + +`docker stats --no-stream` собирается примерно раз в 2–4 s параллельно нагрузке: CPU%, cgroup memory/limits по контейнерам. Редкие snapshots могут пропустить пики и **не являются точным RSS**. Генератор на том же ноутбуке; polling 0.3 s добавляет квантование/coordinated omission. Это не доказательство максимальной производительности/SLA. Дополнительный публичный POST-message load — [load-messages.json](evidence/load-messages.json). + +В профиле 1024×768 во втором повторе сохранена **одна ошибка GET соединения `URLError`** (1/33 наблюдавшихся операций, 3.03% этого повтора). Её причина не установлена: публичный helper не сохраняет низкоуровневую причину URLError. Повтор ради удаления ошибки из статистики не выполнялся. Вложение `63d3337b-5ac0-478b-9e30-cde11dd2fd07`, для которого клиент уже получил 202, отдельно найдено ready с актуальным manifest: [final-state.json](evidence/final-state.json). Финальная проверка всех вложений обнаружила ноль нарушений готовности/версии/числа вариантов. Ошибка наблюдения остаётся ошибкой генератора, а не успешной операцией в RPS. + +Публичная нагрузка сообщений: 4 клиента, 30.112 s, 2507 успешных POST, 0 ошибок, **83.256 successful RPS**, p50/p95/p99 успешных запросов **25.341/146.956/251.292 ms**. Это один дополнительный запуск, не серия для оценки максимума; CPU/memory snapshots — [load-messages-resources.json](evidence/load-messages-resources.json). + +Графики и таблица воспроизводятся из агрегатов: + +```powershell +.venv/Scripts/python -m pip install -r requirements-research.txt +.venv/Scripts/python scripts/plot_results.py +python scripts/summarize.py +python scripts/final_state.py +``` + +Matplotlib 3.10.7 нужен только для графиков. Вспомогательный targeted-failover driver был исправлен после ошибки чтения JSON без указания schema публичному Client; сервер не менялся, публичные persistence/failover уже прошли, обе остановки затем проверены исправленным driver. Первичный вывод сохранён в `resilience-initial-run.txt`. В этой версии Compose `up --wait` ошибочно потребовал healthcheck отключённых background services; запуск использует `up -d` и явный `scripts/wait_ready.py`, после которого smoke проверяет реальную обработку. + +## Ограничения + +Проверены процессные crash/restart с сохранёнными volumes на одном Docker-host. Не исследованы потеря диска/host, порча опубликованных объектов, византийские компоненты или multi-node split-brain. DB/S3/broker/LB одиночные. Durable classic queue одного брокера не даёт multi-node HA. После confirm предполагается сохранность broker volume; полная потеря этого volume вне модели. + +Нет общей транзакции PostgreSQL/S3. Неподтверждённая загрузка может оставить staging orphan до cleanup. Объекты superseded версии могут остаться недоступными через API; их GC не реализован. Повтор immutable PUT доверяет metadata SHA-256/size/MIME существующего object; runtime не перечитывает все его bytes. Тесты отдельно скачивают и пересчитывают bytes, но намеренная внешняя порча после ready этим pipeline не исправляется. + +Cgroup ограничивает весь worker, а не гарантирует обработку любого допустимого 40-MP изображения: OOM может убить worker, затем lease/retry завершают ready либо failed. Subprocess не является полной security sandbox. Стенд с синтетическими данными предназначен для loopback. + +## Источники и ИИ + +- Нормативные README/COURSE/contracts/tests текущей №5 и собственная база №4; результаты измерены заново. +- [RabbitMQ confirms/acknowledgements](https://www.rabbitmq.com/docs/confirms): независимые границы подтверждения. +- [S3 conditional writes](https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-writes.html): If-None-Match; совместимость закреплённого MinIO проверена повторными записями. +- [Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html): EXIF transpose/fit; фактическая версия 12.0.0 в lockfile. +- [MinIO source](https://github.com/minio/minio/tree/07c3a429bfed433e49018cb0f78a52145d4bedeb), [mc source](https://github.com/minio/mc/tree/RELEASE.2025-04-16T18-13-26Z). + +Codex использован для анализа, переноса сохранённой работы, доработки кода, сценариев, запуска экспериментов и отчёта. PASS относится только к фактически выполненным проверкам. Автор должен понимать и уметь объяснить решение на защите. diff --git a/evidence/audit.json b/evidence/audit.json new file mode 100644 index 00000000..0486b407 --- /dev/null +++ b/evidence/audit.json @@ -0,0 +1,13 @@ +{ + "lab4_unchanged": true, + "protected_lab5_unchanged": true, + "tracked_local_secrets": false, + "scanned_files": 130, + "authors": [ + "42bf6aa|Sergey Lyakhonov|lyserg119@gmail.com|Sergey Lyakhonov|lyserg119@gmail.com", + "e376ede|Sergey Lyakhonov|lyserg119@gmail.com|Sergey Lyakhonov|lyserg119@gmail.com", + "a0ec26d|Sergey Lyakhonov|lyserg119@gmail.com|Sergey Lyakhonov|lyserg119@gmail.com", + "5894da9|Sergey Lyakhonov|lyserg119@gmail.com|Sergey Lyakhonov|lyserg119@gmail.com" + ], + "head": "42bf6aa702dab9270fc13d7b80df13dfca089ef8" +} \ No newline at end of file diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 00000000..458feec1 --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,263 @@ +{ + "commit": "42bf6aa702dab9270fc13d7b80df13dfca089ef8", + "python": "3.13.7", + "working_tree_dirty": true, + "docker": { + "Client": { + "Version": "28.5.1", + "ApiVersion": "1.51", + "DefaultAPIVersion": "1.51", + "GitCommit": "e180ab8", + "GoVersion": "go1.24.8", + "Os": "windows", + "Arch": "amd64", + "BuildTime": "Wed Oct 8 12:19:16 2025", + "Context": "desktop-linux" + }, + "Server": { + "Platform": { + "Name": "Docker Desktop 4.49.0 (208700)" + }, + "Components": [ + { + "Name": "Engine", + "Version": "28.5.1", + "Details": { + "ApiVersion": "1.51", + "Arch": "amd64", + "BuildTime": "Wed Oct 8 12:17:24 2025", + "Experimental": "false", + "GitCommit": "f8215cc", + "GoVersion": "go1.24.8", + "KernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "MinAPIVersion": "1.24", + "Os": "linux" + } + }, + { + "Name": "containerd", + "Version": "1.7.27", + "Details": { + "GitCommit": "05044ec0a9a75232cad458027ca83437aae3f4da" + } + }, + { + "Name": "runc", + "Version": "1.2.5", + "Details": { + "GitCommit": "v1.2.5-0-g59923ef" + } + }, + { + "Name": "docker-init", + "Version": "0.19.0", + "Details": { + "GitCommit": "de40ad0" + } + } + ], + "Version": "28.5.1", + "ApiVersion": "1.51", + "MinAPIVersion": "1.24", + "GitCommit": "f8215cc", + "GoVersion": "go1.24.8", + "Os": "linux", + "Arch": "amd64", + "KernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "BuildTime": "2025-10-08T12:17:24.000000000+00:00" + } + }, + "compose": "Docker Compose version v2.40.3-desktop.1", + "engine": { + "memory_bytes": 8126664704, + "cpus": 16 + }, + "services": [ + { + "service": "api", + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T04:00:04.137137202Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 671088640, + "nanocpus": 500000000, + "ports": {}, + "settings": [ + "JOB_MAX_ATTEMPTS=5", + "JOB_TIMEOUT_SECONDS=60", + "STAGING_RETENTION_SECONDS=86400", + "JOB_LEASE_SECONDS=90", + "LAB5_FAULT_INJECTION=0" + ] + }, + { + "service": "api", + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T04:00:23.843924506Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 671088640, + "nanocpus": 500000000, + "ports": {}, + "settings": [ + "LAB5_FAULT_INJECTION=0", + "STAGING_RETENTION_SECONDS=86400", + "JOB_MAX_ATTEMPTS=5", + "JOB_LEASE_SECONDS=90", + "JOB_TIMEOUT_SECONDS=60" + ] + }, + { + "service": "broker", + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "image": "rabbitmq:4.1.0-alpine", + "image_id": "sha256:8a73e9185f8420247c5b3c19231df18dfac28a8d5706203e13b291934044966d", + "started_at": "2026-11-20T03:58:31.88055044Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nanocpus": 750000000, + "ports": {}, + "settings": [] + }, + { + "service": "db", + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "image": "postgres:16.6-alpine", + "image_id": "sha256:1d04b9ba1d4996401f2552b51beda8187f175c0645c091e4781134fc9c9a3eef", + "started_at": "2026-11-20T03:58:30.504381937Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nanocpus": 1000000000, + "ports": {}, + "settings": [] + }, + { + "service": "dispatcher", + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:58:31.312524166Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nanocpus": 500000000, + "ports": {}, + "settings": [ + "JOB_TIMEOUT_SECONDS=60", + "STAGING_RETENTION_SECONDS=86400", + "JOB_LEASE_SECONDS=90", + "LAB5_FAULT_INJECTION=0", + "JOB_MAX_ATTEMPTS=5" + ] + }, + { + "service": "lb", + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "image": "haproxy:3.0.6-alpine", + "image_id": "sha256:c22c8710886104a48b920306f063401f0d11811858e3c6b9d87d88a7556b2e61", + "started_at": "2026-11-20T03:58:30.862509137Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 134217728, + "nanocpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "settings": [] + }, + { + "service": "redis", + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "image": "redis:7.4.1-alpine", + "image_id": "sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275", + "started_at": "2026-11-20T03:58:30.614258194Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 134217728, + "nanocpus": 250000000, + "ports": {}, + "settings": [] + }, + { + "service": "s3", + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "image": "lyakhonov-storage:lab5", + "image_id": "sha256:88795615d5dbc1a5618ff618334251801720cb8ba30ca34d65d6b0ad508a2afe", + "started_at": "2026-11-20T03:58:31.091098179Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nanocpus": 1000000000, + "ports": {}, + "settings": [] + }, + { + "service": "staging-cleaner", + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:58:31.667560755Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nanocpus": 250000000, + "ports": {}, + "settings": [ + "JOB_MAX_ATTEMPTS=5", + "JOB_TIMEOUT_SECONDS=60", + "LAB5_FAULT_INJECTION=0", + "JOB_LEASE_SECONDS=90", + "STAGING_RETENTION_SECONDS=86400" + ] + }, + { + "service": "worker", + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:58:31.445500705Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nanocpus": 750000000, + "ports": {}, + "settings": [ + "STAGING_RETENTION_SECONDS=86400", + "LAB5_FAULT_INJECTION=0", + "JOB_MAX_ATTEMPTS=5", + "JOB_TIMEOUT_SECONDS=60", + "JOB_LEASE_SECONDS=90" + ] + }, + { + "service": "worker", + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:58:31.518418339Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nanocpus": 750000000, + "ports": {}, + "settings": [ + "JOB_TIMEOUT_SECONDS=60", + "LAB5_FAULT_INJECTION=0", + "JOB_MAX_ATTEMPTS=5", + "JOB_LEASE_SECONDS=90", + "STAGING_RETENTION_SECONDS=86400" + ] + } + ] +} \ No newline at end of file diff --git a/evidence/failover-each.json b/evidence/failover-each.json new file mode 100644 index 00000000..c8f1d872 --- /dev/null +++ b/evidence/failover-each.json @@ -0,0 +1,336 @@ +[ + { + "target": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "before": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:59:21.0995573Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:59:04.166647284Z", + "running": true + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:59:01.428198308Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:59:01.96581278Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ], + "after": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:59:21.0995573Z", + "running": false + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:59:04.166647284Z", + "running": true + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:59:01.428198308Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:59:01.96581278Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ], + "successes": 182, + "errors": [], + "recovery_seconds": 1.011 + }, + { + "target": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "before": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T04:00:04.137137202Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:59:04.166647284Z", + "running": true + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:59:01.428198308Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:59:01.96581278Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ], + "after": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T04:00:04.137137202Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:59:04.166647284Z", + "running": false + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:59:01.428198308Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:59:01.96581278Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ], + "successes": 176, + "errors": [ + "POST /api/v1/chats/6ba7728e-901a-4ce8-99ec-ff8a6495b5fa/messages: expected [201], received 503" + ], + "recovery_seconds": 1.932 + } +] \ No newline at end of file diff --git a/evidence/failover-targeted-run.txt b/evidence/failover-targeted-run.txt new file mode 100644 index 00000000..ae01f1c4 --- /dev/null +++ b/evidence/failover-targeted-run.txt @@ -0,0 +1 @@ +PASS: both real API containers independently stopped and restored diff --git a/evidence/failover.json b/evidence/failover.json new file mode 100644 index 00000000..ba8daa39 --- /dev/null +++ b/evidence/failover.json @@ -0,0 +1,176 @@ +{ + "command": [ + "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-X", + "utf8", + "tests/resilience.py", + "--mode", + "failover", + "--base-url", + "http://127.0.0.1:8085", + "--action-script", + "C:\\Users\\User\\Desktop\\vysoko-nagr 5\\scripts\\stop-one.cmd" + ], + "exit_code": 0, + "output": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 88, \"failed_attempts\": 4, \"first_recovery_seconds\": 2.856}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.\n", + "before": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:58:31.589874697Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:58:31.785178375Z", + "running": true + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:58:36.540744301Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:58:37.048579415Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ], + "after": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:58:31.589874697Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:58:31.785178375Z", + "running": false + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:58:36.540744301Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:58:37.048579415Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ] +} \ No newline at end of file diff --git a/evidence/fault-environment.json b/evidence/fault-environment.json new file mode 100644 index 00000000..eec0df19 --- /dev/null +++ b/evidence/fault-environment.json @@ -0,0 +1,240 @@ +{ + "commit": "e376ede0a22aca6a8b1cc80059194dd1635861e5", + "python": "3.13.7", + "docker": { + "Client": { + "Version": "28.5.1", + "ApiVersion": "1.51", + "DefaultAPIVersion": "1.51", + "GitCommit": "e180ab8", + "GoVersion": "go1.24.8", + "Os": "windows", + "Arch": "amd64", + "BuildTime": "Wed Oct 8 12:19:16 2025", + "Context": "desktop-linux" + }, + "Server": { + "Platform": { + "Name": "Docker Desktop 4.49.0 (208700)" + }, + "Components": [ + { + "Name": "Engine", + "Version": "28.5.1", + "Details": { + "ApiVersion": "1.51", + "Arch": "amd64", + "BuildTime": "Wed Oct 8 12:17:24 2025", + "Experimental": "false", + "GitCommit": "f8215cc", + "GoVersion": "go1.24.8", + "KernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "MinAPIVersion": "1.24", + "Os": "linux" + } + }, + { + "Name": "containerd", + "Version": "1.7.27", + "Details": { + "GitCommit": "05044ec0a9a75232cad458027ca83437aae3f4da" + } + }, + { + "Name": "runc", + "Version": "1.2.5", + "Details": { + "GitCommit": "v1.2.5-0-g59923ef" + } + }, + { + "Name": "docker-init", + "Version": "0.19.0", + "Details": { + "GitCommit": "de40ad0" + } + } + ], + "Version": "28.5.1", + "ApiVersion": "1.51", + "MinAPIVersion": "1.24", + "GitCommit": "f8215cc", + "GoVersion": "go1.24.8", + "Os": "linux", + "Arch": "amd64", + "KernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "BuildTime": "2025-10-08T12:17:24.000000000+00:00" + } + }, + "compose": "Docker Compose version v2.40.3-desktop.1", + "engine": { + "memory_bytes": 8126664704, + "cpus": 16 + }, + "services": [ + { + "service": "api", + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:42:15.518481555Z", + "memory_limit": 671088640, + "nanocpus": 500000000, + "ports": {}, + "settings": [ + "JOB_TIMEOUT_SECONDS=5", + "JOB_LEASE_SECONDS=10", + "JOB_MAX_ATTEMPTS=5", + "LAB5_FAULT_INJECTION=1", + "STAGING_RETENTION_SECONDS=86400" + ] + }, + { + "service": "api", + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:42:15.729148361Z", + "memory_limit": 671088640, + "nanocpus": 500000000, + "ports": {}, + "settings": [ + "JOB_TIMEOUT_SECONDS=5", + "JOB_LEASE_SECONDS=10", + "JOB_MAX_ATTEMPTS=5", + "LAB5_FAULT_INJECTION=1", + "STAGING_RETENTION_SECONDS=86400" + ] + }, + { + "service": "broker", + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "image": "rabbitmq:4.1.0-alpine", + "image_id": "sha256:8a73e9185f8420247c5b3c19231df18dfac28a8d5706203e13b291934044966d", + "started_at": "2026-11-20T03:42:16.413919787Z", + "memory_limit": 536870912, + "nanocpus": 750000000, + "ports": {}, + "settings": [] + }, + { + "service": "db", + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "image": "postgres:16.6-alpine", + "image_id": "sha256:1d04b9ba1d4996401f2552b51beda8187f175c0645c091e4781134fc9c9a3eef", + "started_at": "2026-11-20T03:39:52.15355593Z", + "memory_limit": 536870912, + "nanocpus": 1000000000, + "ports": {}, + "settings": [] + }, + { + "service": "dispatcher", + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:45:44.20441819Z", + "memory_limit": 268435456, + "nanocpus": 500000000, + "ports": {}, + "settings": [ + "STAGING_RETENTION_SECONDS=86400", + "JOB_LEASE_SECONDS=10", + "JOB_TIMEOUT_SECONDS=5", + "LAB5_FAULT_INJECTION=1", + "JOB_MAX_ATTEMPTS=5" + ] + }, + { + "service": "lb", + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "image": "haproxy:3.0.6-alpine", + "image_id": "sha256:c22c8710886104a48b920306f063401f0d11811858e3c6b9d87d88a7556b2e61", + "started_at": "2026-11-20T03:39:59.703127904Z", + "memory_limit": 134217728, + "nanocpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "settings": [] + }, + { + "service": "redis", + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "image": "redis:7.4.1-alpine", + "image_id": "sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275", + "started_at": "2026-11-20T03:39:52.151204203Z", + "memory_limit": 134217728, + "nanocpus": 250000000, + "ports": {}, + "settings": [] + }, + { + "service": "s3", + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "image": "lyakhonov-storage:lab5", + "image_id": "sha256:88795615d5dbc1a5618ff618334251801720cb8ba30ca34d65d6b0ad508a2afe", + "started_at": "2026-11-20T03:39:52.148504843Z", + "memory_limit": 805306368, + "nanocpus": 1000000000, + "ports": {}, + "settings": [] + }, + { + "service": "staging-cleaner", + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:41:58.299334822Z", + "memory_limit": 268435456, + "nanocpus": 250000000, + "ports": {}, + "settings": [ + "JOB_LEASE_SECONDS=10", + "JOB_MAX_ATTEMPTS=5", + "LAB5_FAULT_INJECTION=1", + "JOB_TIMEOUT_SECONDS=5", + "STAGING_RETENTION_SECONDS=86400" + ] + }, + { + "service": "worker", + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:45:59.755318954Z", + "memory_limit": 805306368, + "nanocpus": 750000000, + "ports": {}, + "settings": [ + "LAB5_FAULT_INJECTION=1", + "STAGING_RETENTION_SECONDS=86400", + "JOB_LEASE_SECONDS=10", + "JOB_TIMEOUT_SECONDS=5", + "JOB_MAX_ATTEMPTS=5" + ] + }, + { + "service": "worker", + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:45:58.61301125Z", + "memory_limit": 805306368, + "nanocpus": 750000000, + "ports": {}, + "settings": [ + "JOB_LEASE_SECONDS=10", + "LAB5_FAULT_INJECTION=1", + "STAGING_RETENTION_SECONDS=86400", + "JOB_TIMEOUT_SECONDS=5", + "JOB_MAX_ATTEMPTS=5" + ] + } + ] +} \ No newline at end of file diff --git a/evidence/faults.png b/evidence/faults.png new file mode 100644 index 0000000000000000000000000000000000000000..de9d7ee0379ee39e40d4b13a25a61d261aa102f0 GIT binary patch literal 61476 zcmeFZcTiK^+b$dwK~##pAORbQG(|cDP*g;vh)ORiy@nP_sG@?6-g|G-q(dMeqSAYb z5JHg>AoLJI=d7*rd(W9~zIV>dne)dt^X-{u9z&9yz1LprzOVbbu6w;#QIcmobnXxg z24hsXe@7JtqmzZf_H`Vj1Al3_Qbh&-lW@4J;qU}$?BMjm&IqRX!okK8>0oL0lEcx+ z&fW}ZeUtyjb$(G^4pRpQ8+!=>0jvLff*)yTBJil-(o=922Alhu_AuC)80bG51=Q;m)n6&atxudWWaIt7}ULg@f3XVwTho`?`*@b9{JrRay4&W7<>=muuk}$@^Z? zQ2HrZzgSf61SWGNpWxotm1-)B_T#`F=a|^RNiTR3MHP%WpWJL5-_*3|D|VfabtQ&J z6)lM4UucG30oU;Nczl_&>*U@G*g@fa9D6S~W>{b}doLdQ!Vd2J`<(=>?B2g+doIBC z?Y&55qG5r_#xGB$1bnC)yYPPi^D}kblhdEa#U-8Q9jrwUpW=Pz zfMR*`kss6G7EtHQY=XF$?7-S%^|-8Y>y!JT&NWvP%1m!piafKz{riDyYtBR@Y#R}% zUb;Cb=5u0vSZ38O;OxzN%RIT2*z+*2Md&&jZgzEbjc4aq+ezz}T1Ht8Rf$_Bs0Z!} z?Q4)(zAXn2OqlrLqqw4-Mf`j@AYjdPx@ng9X-q zzl#CG@$08Qo5eiO(@YK&sV^+;VMNnueYXn&)oT}Z>t5~i=1HPSeBK=|F@z@NXqUAO z4ts3Qbyd5&mUY|kgAo@&d%+jzFdy($eHKxg(0l=WP1Yp=m7W> zhk%&<)6EQ`khp?pDUx=7B<922szxkV@O805(ZZIbWtQ zA4b*(S*mG;eBg>*CsG2WgY$GNqm!sNExK(-`+6T z)iH`jqEbki4%1z#m5x>ws{^Wm?d)ukl>>}WkxT?1trBx1w1nCE_wRSex>JF0 z*VWl24K-ZBJS&DUQ=LSr1`k-!^4E27^h##C!7yp3CtZ=1Te~!8UGJQ*xnW-xcDLgA ztkaB^@25{k&I+4{7F*%f_{&H3y?_6{u~>h-FrQ7@?U&_i#TXG`#^+k>d>JUsf|qY` zm{@7|0p5J;1u#+}x8EL$LhGQX( z-ye8d1P3?E0xlQ0bIR^CYKAa# zUAfYz>qMB)S-?m;xGt5}Y(xKJ{1PtmX<^9Kx z_M6+ZXelyv>p8BqFJHgLax>6Ed4F6DT+Zjn*_ZxDCz>Nfhu^~mw2MtIv?Hjo;yY`Y zIZiIFX7Kp0f$3i$PQ()*V{yZ5BD&0)q?4cFN9-qCY}?4h1kY88xvVD{u@twNtc*dW zA}hRz%XpA{>aEvwldjzh%D7!Dbtq0-wQ)VVswJsZ=4(wnnIcE7(fF!;ck9oTd9YpTTqrZ=*mWcUwy?ru;1n)b6MA#1iv zb({z&W69pRlsoT_spjh;o(x5heXR}H-4iyZQn+Gr`QuvsO<8;>#38(p3{Q58or`sQ1r@xcORrzUqsLF+$ z1J*`9>GPu~K|SJvjaW|5C3#G)zLr*WZ;1u&Jg(a!A3JGF05?QcixIJs3TM0iT=pwz zp_ZP#rNDZ316MN`9w%m-+#ik&>s=7bE;4D?0>NihEAQox4}DExJOqzCL~BNZhdIHn znGa6v1jEOJKoI7nyw{ahSsXsiU)&VVM=FtiP0#G8?FeFs zTAT#RZi|VK`_Of_JH{GWQLtU^k)EO5M=AE$Mwe}7k`&7_K+rSEP9S**Sq%u03Y+=K zVgYz$4PiEaba9=SASqMNimbn*E3gr7t$fp@tr&#G8AtKoV`I78PtwcdYDiLp;vQS& z__=;cU#i0KB0>eZ;Znf(+IFdVZ<7QF4)v(Txw2ucw}($VYO6(P2KZLT)_5n8hNV3a zU>5YxjBaex+DjcChZYE4u}l<4iRxYnyY{TGFGn+EthCZ?-7Z_F!VbDklOgL@aFfEW zF*JC;MQW2mBBBOv82l8Ik>Td%#&0fEk^P@#s$2*_`G5RabT0tTw&M8PdU(|cyKc|l zUt|){TWt0upIo`M+?)#r3B%_(@|7n0MV$}4d3Cy*tKQ(9Ksl4oZOw*UZ7nnZG*72rJRrsz1Ae;FX~D5t1_a4R3l$zR zL;i35&IuPP#Y;`Ek_mHldDZKatl9U2F2adytgIH>)S6wC_Uf+!19594M&J{xXxsAm z%5-;Mue$zR!Im&i$H0ASslJVvwABi-^Cy5IIDSL1nHaEk^wK>s{RCl)a5AO9pdQVe zFS9jAm)Mo|P+I`I5=;S0R>PN>UBuJIb8VEK^okz-#ky+g=OfPt#Yr9UHA*R-Uv!Xp zey&Ley#`$uuzhmO2t&m)Trl~Pu zB3+dtU--3(8y+YrmHM9-e`z&X-Wh%Ug*U?|?B;wW9OvE|Eu8!N$Hy4Db&w093Tt*Y zvU^Ldw24Amr#)NaBypNM0~YzhPw73%S%8wp>>K~#Mp~U-gwM}w03ch8yE8>dS?sQpc zu#OeO2UsX$0Ww;ATE#XVePGn$bKUawu(5t{kjRBm9fjy_x>jcCwXc|&&)GT8ven*7 z!HzYoruh9p^A_$u2ZqX_bS%{3>r?8)=V#f`W+;8nopoZXbRtQUnnud2Y}2G?{AoCW zVeid+TqiVtcTK`dlirD z`|_eL_LVf^6k5V*pnm1h$!n25LUF$QDRep%$sU(qG%Ystva?qC_QW=0?as09t8@6) z>a!olG&zXAATC61e-}O&gJOT6sEAU^r@4e>xfSsC<7ojAqIhlsoVrp1!l?u`AT;zu zK0Xh=u#!$gL%>lgdks=Wa%PA#;<$LVvYH%9mk&v$fc5Lcuxk^gL#5!ciS2jNbYC0w zZQGbxpxxG^xOP*Dp4aXp?j+VbBZdwX#g*gBQ0~7-Kf7;qQkh}*vy`NoxD}mr((STofb@DR;+!RY#_yD|;bgt6 zZ=$QaC}~8o?OX#-pN!%7{Ho;#>0bWvt`2Z?6>iuLqxcK&(}oN4ka5&RM3i^Aw=wOT z!qROPS{H5L3DQ4j7Od4*sVViw*cxKIYizv=q7+I2ZS8#v{cK1Y7RITmiXo>{YH1*hh4K2DX-wTE5%S0Weu>1t=&I}WSN zj>TE?U|P$qFU_|()ovNnzzpvJcglE(aC-RztK>*yBcmqCCy>*S!YF*@0EqvX}y=DyF*n#X!a z=KHWBAMlJ1;=K=Va1pEk|?ELpYzfpMI>8tn#<2Yg(?NG!9->o-W%3&7 zWFK0%ywa+tI{6uEwa~kmi2WSs?(HF68x_`)x$M`-@=>|~vJK98df>^|?$uPk&#OWU zSZe%)fYsCPv8Z^~lE)%ER&PqnP)_P3l2ZykM%Ndb+XmSgKAX=2OkTya+nHab_=*&g zY;}kec^R)srP=De79(}XarPFOXjkWC#=KP%v#+ge1ih8YPO~95aw~unI|d6p?Na1V z_~i{Oo`!$!?9nO5-^~kb`u^{S(mO|D+%11q*k^!uwdB$vASLsargE2`SMkV zLziL6G&Y-Mb!icLGpkOcvEufzuhg7s5W-W&0(B?Vbw)+j(oT>5x^)DbSQ|zcI{VRH z2sZhkePRQ%+SH}@ctb;A-nO?4sZR#4TboL_O7#NA{M|owuWwfBTd*h(cXNEH+?Tvi+Xz%}ql?!W}_jbXR6Krb|csttIn_gE`*of6~o)P!PTAa)YgEG)h<(PRf1x z@$+Z&(%Uf3+fEVGTlJ-SH6B=(ttOuQNLv>~=Z$o-AltBJJ*D?XIVxvT2 znwy)KqGC^*Nz=Es2Zi}F$k>d?63=XIuFjEN;uAbJvdEh~=V9|xwnjW(s1c}I^w#8bS}F6l&c0?bl+u|ZAG7)4Bu}*4)A=bu^R{=gt&unC z*72hg5&9zQJcV64w?qjJA@wd(OY4Ib_LBwitiomY)S}s~zIxG2E|m>?SlEU27w<&7 ze%oQ2Z}J!dmA6S=c#gR7teFm(V3@Y~Zt|wBy!~os&F&7~!oN5NAO%sf>pcXy<)(Fe zf3)&lTy~eiF1Sd6okCB^3dvJ$Amz28)_$@g%lY#0{+0Gmu6h)~V1kzbC-E-Rg*+K` zIq5;jP4tF+k{`$Xxq;bx7pznqaOE+$slpj{IBxuN9)r}JO#INX_jDJZk`!3g+zZTm zuZ3`^vg#DyddMm@YgEiSvl0-;8R=j|uMQ@P(M}G1@UO@F72-`*6*t?9Yi2QK915&c zZi>?8`L2qISqfCKO%0LB0P(xrc+KB>_`{r;IgO zk5hWC=PMZY248u2IiFn0OQ>`brDCks&pG#zgRp}jYo*}t-A&u{VRJxjd+1C%Ci?vx z@XGYd8qkp5cR0Ip2QU)?@K4ydH2TJh~W%8>fv-y9{v) zcLLa}V>8?l`je+NMZ@VfXMIrWt4LnOJGuDeN^+!#qXozJZl{bI3!$r1bGP>+@e0-* z+k-=n4mIRbzJXhpa7MO--QD$ePxKMHDRjqS*L@|rW$zNaSX@`=|WJDL(j* zbe0J8%5^GQsRez$_%$owry7cxW{DWZDL#upLr_W;THx}|@8{U4yzi;(ylM4&VL=3E zA5%ttQD#ku9Ztv~C()r+Uc#_8_g|}^FT8T+{X(^6OpKLs>6)YB01mXb4UydFM>8#5 z4zAIJaSTB0+MQ4l!C&gu!=Ns*W*wN7fsOt2YhJA;{QB%?J8(ZJ!W{s_$wO-qdg#Wj z+Mb_m%IbJ_84sz{1h&tVr$xrC=QBEg5&+6=0+7$}+_qAz*r;V@vza*MgBkXmN(Q@- zX`q>GhfE(-kgpreJzStgGV-hzx{}gH6adO=>HYN}f31)1BBLRK((_~)uql!EE+9$O zk5le>xW|iZO`lwKY-imbo10EoC*gJShgO`h&RU|iZW%k)GV#9{xtpp#4;`H zqV#Y>$^#>b>g^VFVIIB%&N=EcdD>-4-?%&GxI}RQJP0)x07c4N#t#@Zg)X7r=8`bW zC{2gWvi)zX^DL~cw+wu=?JqP+f2^4Ap;vB`)XzWXE}gO6RFXGT%|loaejdeIH8-3j z7XSA5R^ah%CZTr&AW_&KsMbTp2L~VPaFK`{(-kw#XR@m5rL0hJ`X*=rZtli@gF3>G ztE{5d4tl-wbCumOY&uI0{2x}(1ra}X6vwSJ;jXgD@$>D zuZN`Tqy^RhH?yr1!&cSdUN^6hL&Yc+ztC?U=r(1kwN5R2eF4;iA_x@(!(^F)QgbfC zQ`a@zBn{O~Az4?T#8<=;X|%U@lW3F~Ke)^ok*3eEwTK*R9Oi|wSRDi`-^F|R;n5;i z(JncR@EB2~hwx6&-akskOG{BNJ$jBHNF~yz))v;SHU3qJ0%4hxhiEXb0UX zz6%#>AF+S%iM7eei1p?=&ZQ*eSGhs|I3L~p;?k^4lNP=1JuBHVBV8q{oE{#&T}r-Z zRQ_%q8C;?g3 zCd{HoXEDq|2Pd;G0H_;GX?OoJoR$6D8=0rBK3U8^M=jC|%JRF8KGK;jP1%%kTl*qT zhuB#g9)PnT)(iW!g+NxKZD`fx<>pC8$jn*+nB{Wd9dpFdR9JYR{*42J4Db|D&ah{ z3*X^+Tp45sy~?X{tkESmaD?j`jtk5LJyhk9A5`MB&WUNtv4+BOc!a&haFGmGWc!x@ z@mKJSP`$0hSG2ebd9$u_Y{`fVLhll56%cvTNNpVKUAz;55`1{F?7e6;e;G+gGh!25 zo0DE@8dfO%QPfLslXD|Q0SEGGPD`<+yk6hAxE^x=maI4gKf(rM>4l19;z`|)r}&y2 zdNa5-Oj1GJ)J9Bu5Y`k1ir$}}SNqN+kKMH}@aj|FZ82oiyJ%tHk~z_bMgxka6-$>} zT%evW(%RI0vQ0!+G&)0;d}QiO8>V`ty_5-`q+YuXXekkOYJUaXE@bC1{Km^S(?3#f z!K}XYO!UYk4rC_#MWV0d#${iOL@jJ&Xb_23k#DMY;(WX<48IbHiYtk=HrS*th;D|6oWL}Xj?R^jqTLozp6^ms*xJ9jcy#DplVGO|xh_Oq z*d1g@rGKH9gq0M^OsTI1bXO#t<=pkIE%pblzs&=5oQ4zVCX40YWVq3-za^Ds)>LdE ze2fUxDhA_!&n$v3ax77#%7R0!Jp`SDo=Wl+oY?(hCcrYsowS*DDspRRn9(GPqKQI&73(Dfe zM}TuUa`z>3N}}f@g5QgH?c!Hz`BmM9g9Ddt=MR(~51RSH?YOLnLoF=W92j`gpyxZZ zZ2m_3YEAd^a?TqizN3qGULVwKHcV45p4rK)bDdt4!^BE>V7qvl$W5!qvnUWhijJ!X zl)7E4e4QF#+Y9n1%UoZXvh->5i{M^lUv%L= zbP{)=e4i$qO;yR2W8FUX$Qx_`l7Q(Y-K7KJFy1%6o*O5aH_PD@&m+8;L(R*tl0R5Gi0N4z-y`AQfmvPt~PsT{zV`70>T@yHVjUW*J# z*XssTRY0k?ekry&%X7uyh;y2H&VZ*DdEd6d#!9zxC7Z#Z9(CZ%bFN@YNzmQ=o*oTlRTJh^D_T>KI1dFr!{s%TNWy9U8)Qnw})B!_E z5hdAI_Rg~CVc59*-{(dcpIdShzG0%|@X5{f>IIzPltg@q$QCzS1YSGExo=bz5Wd9o z=2tR!-p|R{NI1NioNIWn>oyg5R$SLM)AI;H7(UhO6g_)g$6Hf*=S#$s*H7nvTEK4_ zf5LIdDtZ3a$?q%9cWdSw^41@kNynG@FXhuQoj3KtE};k{vGxb7Xq5A1q=P&0Y5je- zjHG1JmxmGjlS_avoGp!bBXVN8eA`iAA1*rzwRUjxLtAWhosFe7K(P-(5JZRAE}v|V z?G14AiNCFNY^=`oc@<&f%$QFotL10}O>0#?HIth$a$H$z z)E+6qX4j%Cz=FND@tRrsdVnNqCX{KI#e!AZjCfcj}P3GWxZF_b@{GQZmz;1gh2w+}H=W{$S_mH>m^iZb^0D`|F_wH_rcpMYNww&-h#M!1%0`pn_Wv>}}) z%Lydu9J`68D6@Fi`3g%E7EqWiu5$w=Mf;9Uq3v=j`SV4!kgfv|B6FJrWquA&cr^p0 zH!?MS0ZPxRr#N)pJsq|18^DFKaDH8LaOU>VT~O5KoYr%${SXi!IspN`tk}NhZYf=N zsBoO#N+V&xtn33ty(yo@m#-UuhHb%gTglbb0X{rtT`f!X#{&h0#;1-zd54qT2ES@Dj(l0VAkbP#dmx&xLqnJaGujm^O)Fn>t400fg(S1}I^QfS6(h zQPJl_EF)1%KTd1MNjkNu4e@6;&%ZBQ5c{>zgOm96Sv`N=mi-q7mHjOF4g*mnhhz)j zeIJm`T3562{@a?!s-=7Bg%}<5>{M6UM`84eDG3zAq7x3R+J%Pidn=ugCjIqr*Osvz zVzUR!%$z*+p}TSbGsZ(1C|6QSX&D)k8QliK*7_1wK^O0~m{|1}{-~-3RT&A~xf6dK z;POLr{qr^-1MMLPNT-^BiY^hc8ZfLT1GKFLhSW)kJX_R`(N)7){lxD~DYnNW6kP~N zA7*yz8DoKZMNxvG9!>%+!8w4s`Xv2R#q`p846?dlXJ->>U$qJH#9!$wrT!#h$2Hp5 zI%K{)PnqC}8t#yoS(*uwhZnqj`7*XX8`6}nWe zO)V$V-m_y)fQR5XYk=Wz=1OpT8Lk*%3p@prQWgR=8SW4In^~qR?l!x8Hn{qSk?R4P zwlWdvAI77uWzrVg=$NPc@S**V2)AZoDy2bsgKcF747o6?pzF z@%kxZ|FB=YQ<)Xs74Rb?Cpv7wVJ^@ryhgPv^(7JAVy^(I7 zS$BFAjzT2>A|O1g%KGN&^3)uVV6GES5Oz_Un7LAXZ2eS-kD z#o|b9)V@f~vZKf{pczEMoDT|a98 zGDOFM7=U;rXI{RKk^hrfS zIsp&SZrobMnf2?CIKGO}U32xfC8!sIcWx#f@yY~wSgPAZ7&$bUVRvq2vD9NW2A)E; zgWWNcD;KCY0@uaG!}zKHbZ2=baLw@ikt*Gm6#^0-=V}R)E8Fl(YVB_^gh_|_&XGb< z%>Z>Yjk4NUeL%aAU&gRp#ZlPfa|c%)W^ple_g8KoQpWpQ*jywj?sq>IXjfK)+3~;t zW);yU*UrxFz7mArn+7*%Pr*2F6-;Kon-|sd#t!4k$j(RJ_Y0?ℜT zmCE<;UmrVbp-us^mJnVF(4iJTrTZ=<7ay*OGx*9jcP(G;?vzG@#2asaD>txk8Wufr)nv0%1TLy*=xv~(Pn zSgmBbo0yPQH#60-FMg`LQ^N8Q;E~3=9tsFkx(o2A`#!+y(RCy{vF>eeuC}^ejJ3VQ zqwhd~w*E=RNldRMk8O}x%pm8^S*F@2a2mc0l3v**U3QF~!}imBJ%Z&EjODyfcIOKp z2WMf$Zuj;1((7^EwJnt zyji8xm!#w1%*5ucRKrCWw&?wAv?}Mii5o59REoU{(yk*XLmLl10B%w|AN@{`G!re! z!9zd_WdUZp+I-bEu0|<;ti)+yh%kPwD=D|CRnAOYHv;X}eVKq!a?mB&KCq{oDxI8m zHe<#ob>rn0)*l8?B%TH{nW?4RVGh{n9!m(Y{mwA&{cNl!;YrXF z3d-U1VS2-#z;r9+{VY&=(NAP_^U?gB2Dwa%X`8+oNhJI)^6D_!@;amd|LM}6VRmG$*K+DXNV+cl8{J5o2hg* zJ-b`S*lN;ne!PW)CaSU{$sSdSGl=AL#kowZY%2J+Z7+E4TKF98ZWjQ#Nywt_5%8<= z**_u+-O)((EP}}v+nS!Z+!Y3_aWEE5$kiueK3%>Rf9xXh4y$!>tr{y`g3ILD_q0}t zy*`($GP@_dT~p)(!fE!?MRIuib0LWXw}&9-mHjJkk>BPixn&=P!KnhzB0zlzgS~mf zDbqLNyLx*uBZBUtTs~Y4eNwD8plTO^U2fu`l>r~wg1mY#nC$&28}=ChuN!QgS?+{0 z(Ow5)$<5-X{mfEdaY6EIl8%qma z;0GYqZuITxLi4>-9~b~YAXO#lw%x{xSxxgXSfhc^T2OpEeEL5hVX>_(jyMk9n2l#| zY-|k5gRHB6j-z3Lfh;zfc3%Hp!@M%({tVWi*LT?dk8Pg+&dY*YwuBjOtu1wBD6^aO zWCrOtkG@p}))j7DOfh6%((_n15(l^`efp-Lpy6~^8uY{gyzmPZ&>srTQc3dGah)qM z_6Rpx*9`y`n0Ixfi|9>I&dHUJ?_E2GhHbDsB%X67+JdNizdKzq4e$bmp1U6F0B0Kk z`ArajP))!YKLt?vQH}eiqZtq?TynZakV87aOk|;#AnBA>;SQ_>f<{dz-UkFwE2T@! zdj(&ebo#88dv3gI&&2Ar?69#kAfpXeR{a~1Ef=PtGi%+BI z#!cx6v#!+l6~jd)%D-j~?ES{FkCY^BI2Xv{p`&%aG4()0S*jQW;y{Z#7@++dgSpye zO@SBW#=pMWuL^*@Sx4flbvMOAV0!&~EQd?6!#ZE&Rd3AjG&VHYaNfdkykkO9KtSIR z3IN`S22kr-z(GhyB%*mW3%?KQoa9#hPHOci+4WdP@2hE#4hP@Zs> z-AAJ_`eeoX_ZQcIPPdr~9A$Rn4M%Hc_I^#a7+h*#WaQUE7ct)E!prRU6&1sP7}e}F z?9K%_iNww`MS+4#2cTcQX6)MUkI)3{);5?Qtmpz>j=)}p*VloWw4C9?>_AB~E&uJ4 zFog#Xg1z}K)dd%sO{-q!S9|&Ld{ExrFmXhfNgQJIy3EBTkA!{iU7l#3_`s8Q{T1z; z-{WG_&hz_E06TB)SrMMlVxsRVi@3v7An-*gp{et1D^hbK_c`>9g~Z~6uRT%cYJPLF z(ulnCqzy+>srjk1db*e(>`P0~6`Ql-J!%)ARFn2k@!A4KB-DpBSUO*&ig3ND%>% zCh5(-wvzu&F3aCU7MGRc)~hP7fE3_T>X7HIy9-!oakT+$agx5daL9s11#Yn;c#}Jl zjK)Vct+8$emg@f9>8a_>*)rCl*m~9Rw|SIS71O=Pf9EbU3!9xe@Qnh~ATW<&MS;Q3 z1=R20q@F-mc#&N{)0=HXo@|}81=f(dRq;;io^Nx~2V``nSG4wq*~{%XtpCf4*PqyB z^xEPPF>?q;u|bf=?B)l>&3ZqF30eN?SRVof*TwzxOe>OWNsT?E-fS+Mj=f?!u02-V z038CPH$m;FyB|*}Q0ED)V_;&jz!)wP>nq8nv`^KVaunve1=k-A%f9>5?3O#rUx(V00{M0SmQtMIu%eSiX#w#~UXn+ahlm z2qI)aL8Xu$FFcZ*kcCwcig?#|>Bf!6z?1%ng%c=Ik5XtN6+$@Q8X->5Mlb64mK=H$ z9k?~LKn+|Lnzf=BI<4-=dDv0Ty?@aX~tj9azTQggZ^>GkycE+Ck7 zkI}x0`ts6AC3`;mWF>>w-g7Io)o|>Axp=QSJ_fnu*j(35-YiNB^Q~4n`$QzG?d$K~ z1A)5n_efT}&0;m_B1nJmkk3tGw|Bg+&q5om*qc-T-seCch28po;v=&Mt;igWYL1HK z{h7Uyeg~(OWw}vsgDFe9;EoD8)BXj6PtD%V8-gr#6eZvMJ+X;;q)=+_bQ0incL;+* zw?5E`y$K7itmxl8HuCRH*vJRi!Hb_AI&Uv83ZwR2%bs{}lKaH61W4k4j%AsB9Pi3E z@x7r#xt}+m-OIq)?R~TSIC1qE$7%8NOfE?l?pdnonN^a=kEyBYu57ChTs=$|<(l`- zGU8uX$lE+8`R?7DH@g$X_m!0R1Bbu9FUk_6lo_s@|FG`28ws82%d+*CI30eSp||Qr??}EO4ggLf*#LKo_|L2>P&x{1taV*5ZV^ zA+|TE&7f$z28AZj>hTY^U3pSbHiUi%)c6~=V&}VnVM{&8BL7xH1g zI_e3q#~lgB*{gs*3j$&A;cGg^%iP=wP_+B;@w6yS1c3W*4J(AdBm??bP7o>;{v6BM z&j(t_0IveJMm0l;?f9AvYHjhwbYE@^gW(`>`9SE7=xWiI^QYGY7-w4|Z^-3=@{)|9 zh~s4?;LyfF91jARN?Uq;JmirP;Bc=1N;(wunXHrBWm>;z zRIiP^W**ny-H7PP)y9Q(k9<%hrXijJP|KTg&o3C_d8dpZlhNNpb`iv65&CzcwzjsD zdiJ)z+7e_00B#cs0DkVM+(&?LNdt8Ce}g z+B5&V3NM)*T-ZhC7(lGMQF_&l%ef^PFm$&7GP#^zz46%|0}8UpO#s91gJvE=+e3R) zxl+vW?enhha-lv1vXt{;BGfx``L6tke@1FkPj92(3lLwcc0s)rOmY*ASxUpszPO$U;;!FvjJAHK6)nRHStvNJ|KJW9%KFr=lU*zf-Vo%Q*my6K%sFlj#4;v*B>4V*bP!$m*U^$%#m|DBvAZrSi}VFF?wXydai zr9uk{#GLF{Nhc*R#H-&kSj>^gmZ`fRx6(5|QW(|rXKU%49_ zIL=D^yBdz0zDAlZc@*bb?*n#VnZ&%vg9zDuM~LkSqAp@;z~_S2)3Iy3!r2fM85x;1 z3k=>y$I>h~QgtgGA)=rOgz1Z5^4_d7 zr-BX%<0L=_Ux12R7q?bnT!SyW$5jv{8&7SiK*|>w9{Ie=d5Lx4&7T0_Eacdg2Ms`C z=HpS%YdN*Gco(!;wgzesG>ex zR~kYPvWE)2d-mT!xmmnWse9g=t8v>{W`I8xwHNEg86V^J(;ld2<13>6UXpL;Xx_t@ z$;W9}?)0=O9#1v(lG8A&h>6)2*x18aUOWH{W_H~@+*pU}VaUu0=XsW@(m;Y}BpdmgF&1YP!dUj1yG-hTl=rC7ZBE1hv0F{s{2dk~epJYU5Z92`B5`o%N1o>a@s z_n#k;gmPcm0>`jXaR41xTY6K^aPR~G+eXc=o+N8${1l(j(fD>*3Kz?M#C5NJKOhG| zYFgRlNN+gIHZ1-{)DXu2>5_=g(B1@A>9|bWY23!5(NH8b^*H%@%+b^m?)zXdD}MW752cM<6w+#BAJ0I`~Mx^J`24?aKAjpIG8HXGi$sVuIauU+tF zuv73cM)YBz>fJ+2pdI$_QDvtQ{$qmTavwm23S}jwaPK>!b&n61HG?|r{sVJN{q%=Z zAHsa|vo2A~vbmXjFR7FET8$fItR`1rx8e_a{8!;=ZZqD?F4{C?z`cAMw?TVr?`N!o z_R}8CK*+c;c$9Zdmz>}G`#cM%MC?7LzNo+IumjJ+r%LJHTV7iMpm2C_Jl?f=ebf)C z#A=(u_U6|jBQ%RZf&Tj1?z6y&y{OsA&3}R+2hcNIzzrG(ip6FGb?_2E9n6YzEWKc? zK$?20LjDDc0nl-E?D*J0P3+#49LS_^021~E2m-B>i6-dn^cd^lo$bwLuxNN74x-qa zprxv+`t7m$ov6QMmC^fH2AhL?fUI2kGsVlP*xdFN=nxK zrcq~}uE*aB2ZM0nHQmj7yzCc{`5xM;T~*Au&kH063B)ZH=$fH30vH7~Ym?Uj&NTvj z6z9n3XW*0q^ifbp-?7J}$#e!)p|2NmiU(U;kKNYe0xY)sJubiS_c5Fs|EUtat}=UF z=HzSw+*EA4NqamW=s4})#{mG7aAOyZeki}zeRy@ck}S`|=CEt3?@ltpV0JC*OPves z2a3as2OG&;nsv|xv9!_z0!FbB9+N;oyp1l*)LKFV$(+U8ZPq1zLr&LZE4C0;qi=-*SHVJAewu+6XYx&h1_>JuPSemh-WqaA>rZfDIcr<3 zlv&HnXyXRj^&no=tiLdAWu73}EkwfQM?l@%FoqFkMq??PXM7b!&ypW)^J!LV6=;{9 z7rPRkw(!2E;>#Na=R9$lGpnF-UAu<3q+1HdpX)hMQ<98^kG(FU7>!o#J?947bPG_{B?J%|7Ngx-z# zA*fRZ5T11|_)@DF(8Qs04QvX4#8Se~i`(7tha+aHut18NM8w_8fD*y7x9U>bcy;sq zVJJ17)fXK9oAB;jhXsexA)qu2l%b*{=Y;Mzg>r{vaWw$THZN3_fJxpAg3Hmo4Jd+n zkcJa$R?5qL^^F|RfAdm9!ufC`&FV+LzPykF3--ena6#y0RL)~a0FD z6y%_xZ@)_xu}`q~(n0k;D+JuX%kTGGmk|gAesg00bQ7APKTH*aPs!?K+zJ3w>G&4( z9WU>}{o_DC4TbFUQ8!Ie0RXhaTa_sQ(@Bdfkh4MoRR^tC2*VbF=G}^wu7`EH)XY1~ ze?km&4sYa5syHn9lz<*DU}c5aLs4gV;}n6J$q0l)JK%rV&^@}v3^v91ci7!vz?cK= z1B9WwfV8~`nhb0_66vHN-oI?n?&AqSr!PF%`!wuLus%YT#pBP7|M?@w>5xPy0{l6~ zPkS)FGI|I=DTtlTpq~bz6cQPE=Fxxh4a|-UR%kcz;4Cov@amMS02EEMl~4?f4-Y@( zoV#}#nB65ro$ z`#FGdF8>eZ2~eks8~@&0vO}QD*S4O`ZmtB^1lp=NKolO${u2?PkH}hIzLl?8lwL6m zR@fMN{lPfe z>1BaxaiQYwYaZa<4rceJ$w=5j|Hf#{ zf6oEVl0YvDmVmK0gXcCSqKN)G=-uYa)haopkOtZ-kk<1nQ=M|aHFGJ!eKiW|@Xkoe z0faoyan48A1L%*OVF{Ml!%SXR8N8;hs}5|w1Ug@v;Zzgc=YgxC8|a!0j*PrI+5;$c z`x;PZCZs0_8hwD)Ht>pa0gd3W1!3=UWTg*znZjF2%f= z0Q^t5MP9`{!0%iEMIe)QnlLx!Gw?qtcLV>+z#w&|_u-+_{O3DNtO=NBU|))h^i1ai zK{;|CP(-PKhEh$s{nqP$dM>iCaX-$L%_*5$S1!Btv5(SVgTiy0uK6C5Da1?CsNDB?7gr`H3>(7Z{6TD~2LMa^l*)*Y(f@I1aLJf09Me1~6~3 zcWnLVu4#S0LK;+0&(j37nW+$G4`^Dy3{9I*K5aE<3IXz|wV(ww$ds;v^Fy#m4S|66+eYki58w;))VRlw~7rO*dJroVh5T{(dt5Ff$;sD|vEI;(yNMnS%xQXDQv^AhAs z7MBoWfVR`9@)!bIT&jMJhbk700u*)z8P)2Ptbhaiiddqn0GN-yya0Nw!c$W(-&6#3 z!35ZuBp3vuhgYH25a{6W)Y|UmKi;#5J1N>5oQ`|O{4fs2QFA!GRVwinop6UOG0y2h zx#nFARyIbB^21)gurh!I!JL21KC~ynNqYiMRUz1+vi%W?Q|+qLsH3_N5zz?4vuCK4v>_m(-?T2|E z@iJBV1a>-bjhzH)emJn<)nFr7GQYdCCHAb2M80PNy1*dIm* z+DavQ;daPdpyS>LGxqf7jeqOXMQfm9WyIv=`_y>mS~nPMhh7$P*k%DF<=;ImB=PU; zDlFem`ZIcPG6mgUx)8|>`??E*0wT0{#KF^dCSCcre(r=sp78}sMk>=KG+RgK{0;xW zUIy5RJS-I?)Ve988n&3t8R654;i$1$9Z`yBlJ$L3&9`1|n6@@Er)I0o1u3Bb&S z3mi9po&rT+AiidS46`!Z-`(FWrym1${D4S*{_3@t5MO#$@bJ#Vf1vr#Hemo+x*1ZG zE5~tW_$_{?ABrWZ)2QA$RW8`o2CGSF(BHvaKAj#j4*HOf`(7ePzLAIl5vu`E@|SPi z=v?^lKiGTIaID(*ZFp&v212v3grbx*kSRmvDIzkY6v|MhGGCRD1~L^DQA9FSD1`>e zSZ0bMMMCB&L%jP*-1qPIJpX5V-|hYKet6fHd%JU8YprXo^E}Su*!TU|&##`-L|cqulIV(mWuV8C854p*~UQG zCA=4Z*DwoZR1ir2eWcxnIimeY9noLl=ZB&!8#N+n9@nkc{!5D#^6L^B2B!mzeIgU} zG%vW{K}pK7@bhe%kjT%aCOlWa&ZnA$Uygjbf+el<<%EYQ{E_SpbC4v7Hb}u-O}zWn zY-UbQA(#^_cDKf(qfr(m{(td-WW&tIi!6509G6I!YupO>-r&KPmmS8_l0MeZ?R=rV zGF13!J#UO{8}Fm_t0Vz)a0&Hf%+G&(w5X}`Mc0nkSBUO5_NhX(z&DcpBb(WwR8B6D z8Q3T5lLckKZs&0`Ad_4Zr4v~`lEZT8sV|fV{n^5x6Y9^%&TcMrIwt0F6<=w};ow@` zU&?7msGb~?3>FrabywEx3`Iua_4q0tiS#RxMrZXLW9R7RSlgSMPaE6uHBg#VtLN80 zLfQNZFT(@+Qa5a{>%Ovcy>)|R{4NK6Q75{+{ZPajBQa0P3b8d1PvSzf4LM~g zuWbZdSufR_oD=EMrrn(rGR0#A_|M`3c@Tt@gYCYGlF~Iu=;t34A}_`QrQ73y13wy% zkENjwEJzXly@NIjTtex5&Up}I#6+6?2IEhZ9~(@vGa<(j$ZG|Ua@^%CzD`|R{HK0F<2Odt3Ex#e9z!it?Nfd@)*xUU8j zBRtvtx~cR!BvNFnnIoQ@^o`kE#J>u~qS5oyU1a@zKR6!gIlciD<<5)sXgpeZ$Ps6u zm0rAZF{JkB@%h{R4UQgHx}4Jz`JRfVg+y#_NuRURtv(~M?rq_r$=g@#9gh4A>!Wr4 z5`#BjL8uSxh!>g9_SaX@$)dFR#V=vYk10{428=lKZ9&CBvgqV5w260Bkt>>=?FD~BII3n z#g#OhMQ7**Hr2%FuBaB(eD#)f@#~b+OMKq49+|p7zrLrh2x?N7Us#UA@ zL4a*p7JcDu?yv+=2ys1B&b=Nuy7`4hz44#)hNEeQ1-}DL<5$o!iNvYTSe0=dpQFg_s**D&XAm$;U5gX>;IkeI7$O_Tl zKW?7A-c`Q+-T)c$6Eb$9%F4=qAbr%Aa7wcIeZs#|+%YS<#a)M=^eR>;*s(29X{c!> zSjHV1gVRt;H$D;IIg5*k@|Dgdon4}b0C0VwP-bcWJQA*n$6;d}Fg=Qi6|yCZUD7|k zh-u7w$|jgrRJlBF!ZxbX)jcV>eN>G7HRGK_ySKW#oNRtMx_N3?9AXPSCOotUtsY0e zy}kJObeAzfB%Q61uM*x^)N?#Dc!T!dtL(dvjq}6Ls-zszmrBq{So_jd1dF{+ndK*DH8wJpUQ5dH=^t9=MKMZ47hQa zEXox1hLeI&GnmzJcxl`^b99yL1J}D5hI1#gJxEWJ>0^8dBh{bn(W8CO)I09{(0b;y zX;QkAFGLrBrYTYz-7ZPHo`*uBnRDW|LTu|kZEdyWDg2{10TFSz;c%Uc*F#d-)7w+` zfF3_SoXbE|obUTi*E{F&h?5c~wl&@A(^^#UOH!uzGCAW`8>8H%Uo{UHs1u@xD{_uB zam4b@B5+FU;$FZ`%E*6X;8PLS>je;#69TTFlT|CoEz?piZQ8HAagiI>oyP{7)6ELV zGEY&a3^a$IrF&qnwh7*O*Ab{xz%G&nm6@f_^9kS#A)a;Xaxy2&cxeya@y-`9RwcW~ zHr}l`A&T$IMId`-=MD@EJbZ3lIIF~t9XoWMY|>I@s87WR^@V{PaSXij)yqy zx_7eUUtk%nbRESkKe>uFMf`hx_vR?3+>XTDSJ_*re{q9>9NT3A6$F^q6@P!DNz!kw z60uX`Ud11G$Rcl%le7;Oes;MT zoE^VL_7H}o7oF@O`K7bCaHhru9+wYD%JABR|FjR|STPh}F60@?MX&kl+8lP=-b8v> zo>!D4J}4a#Eg?0wz=NLr5H3J{r(sS#@@a6Pfd;_Z=?L!T0yR7HWj^H@gfm=(|%*QhZD9=Lg!R7be}4 z4U8D{%s6}aJKMgjJ##JX53+C2Pg0ZvA6E9k^um_i2O&v;o^^xavy`!RlZil7{(GTr zUi0A73i{V(sTioz$L@cK3>4rsT-twJzuG>GRY>Pn)k0P#U6o9%9ths7?_j(t*O`QIeHLiHIg11_T@Aiaan5Fh?$}9W5_&tRhVmC@Mtyx?|%24@G ze_QYqG8@MBi6mK+o+$pis9sz8wNx+cK&aWR1tSF3r(6dv33*2iqst+GJs{ z7B70D-kBz>pjdTGTTNE1St9^Nb12XkpCf~~DO$%QR_J`qlwZnXtcJr<=L8rjdg1xL zKL&4zJ3lVIy<3mewxX`p?ls~*Iots5`^>9xAJQNDZA(oCZ`bp&>{RQ|Q1!p$dc%%l zi2b-P=H8oE^P9e|u^lneEy@k$Im5upV`;f6Fyh$U?TniuuZ41NJ!P1@h|1ugNIM@f z8YQ67Bf+lu4vT3xHRXI^T+)58*F46qr?3~ql#;T6UyW%p)XDRtej6=hwQBClK3EPF z%v}?;?hMsW5vR$eL!a=K_?StWtKYuPeALg&%$TB}qeN;P0`ylA5O;Hw4@V$Tbpg!G z6~ok$&c5^dGTq|&`PM5MctKWO$LuLeKT`LWS%vLe0N!qxS4()`dDCR-W8M`%Ss!J? z=EDC9it}G<-}$;>pRgoP-v-^aB3RJI{BsK9cjrx92Y`5A{dOQq-JCglm^ce@Yt*LFOuHoY=S`ra%qVwXfzZCEa4jV9L{ghLa7{n>Fm)j}&Y^*8A~ zW?R8ATl(!RF4FS#HaE8b-7Q=oBtuS?a}~kEwdo>XL;z{_2@^g_qR~FkgFs;~72guA zu+AV20cQ5~HgLgN6sP`IyBFT`4q&NO=fE(Kf%Aq2fqli0L7YcWXOK}=N>GVZCcQuVEMBBu7ruhC+M};5qf!mDDNTRDg?=VlnC^J zuTY_^Q#V#7Q&-@!$dY?o!-SDjveY;262`?I&rX=krskBu(!9CI%Z01@(D|nY9YX_> zV1K?U|3PNP`&HPYl*#*ebD#&9wT8EhjjhP-h6%ivG>bA4dihvX^g%cO#Qga zXNClQQ+KS>o{?$s#->m_-kUdf6MHEMO*jT`TeZ{1FqTr>1$XavbEMj;69v$yurGF~ ze;8S%8G=IWX0&pf9B7}yMQCGt&abIys3{d!S1Ag3a{q(MHRc}X6$-YDRQ@Jt#RG!? zAlw)Q1LgJS36zJ&I)U8{T76ln_VgWxL=%LQ!Eje*rmz;1``)p-pI^(Hrv;h8Y%b@5 z=?0N6*)PEyuM$6#0&4PY(^x630MIx;3c&u+C&(O+bOJQ%9CRObbe+n{aIOIV*tEorM3|7JXT5pDb6Zzj<>0kFYb^fY* zs)4y_;gURkY0ZG6W@ct88jS)kmA}}RNZ6qRCH!7P=usu`Vei#AZEI5tGLxU4%m6oV z6qd>RvfZ4+5YgY}<(a#@L`Mu6G%7&*Zabbp;}H_lQd!0@OAhybEffoGAKRuCE!5f) zQ_KE0^Bdgyt|+z|aDscn;EhputOiUVE1x`Q;%{0g-$4fp5LZv0 zp5MLtMyVyxY4!&r1v5zcGmCD5t<0lA@RA_=zMiV*;hJ+T&$ zuV~ko7f%WLbc{a<6N}WwRv%OLpeXug=nejJOPyP)xo>sdZL~ubpw796vPT##W!g_s z6e;f87nosLZm-5$Wi{(?jRh6;`yBI%kuHtJ9r^q7d~G-m5%Hc0S>wD6O{*I(=`^ zN<{qzbhMIqOET@_Q@^z(L&UJv3faYaQre@m1itrgGM$j`&h&yYXBWSB?m@v<^yb3C zW^e++$?Ye#^^6=t&}}UtxnBA2a*BeJ@1+ZkYwRc8U$}V^M*YNcc={fwCVOIWGj&K; z_S~3#4|o@E*)aUA)mE^$cM5FUXa_iQu753BrPRsm&wQ5hT(V^!E+`L zu6?NLVH_h1z~~~|eb?Rsz8IK5zOJ4WnvjohjnOf&Zz!rMd%SZ0%s)0=qI{F#-Myiw zrza|6)6}_0qWq0V4u&jde?|s+Av9Nk*G1g1nQt-Ga&SF*^ytIQ)w`XnVrvX-i69vk z4@F2KnqizU_>y<}OjJhM?6=#m`=DntyiRsD7qYVQEZ5V{a5a(LFE#xsTIoD#hXVqn zcp>Zu>g|q+`_q4(yqAg*-6tdIGQ8ID3K=0J{P1E#U0*jYy!{|ZvZrrpNvb=o?oLs6O`m) z=aVUG2%$Ic@2Fpn0cY9iTQ2uAiLiwU zx{1B_m&|;y$L{ejio>R>!T_5>4mpEMj;AeAXoBC+^ zh1cvq5_9<+{U`}X}v zt5^#EdE&pS|b$37c`wJZ+{jN)gnt8B~2_UkK6&LO6QFs}-TjGpC)CGnJxZVr7?9 zHU^}orXFhBf9OB(!pYmBxz!!xXleJ7e5W)P@lOD#L;ygB;x(2tbIP^hf9XnmkLx8D zHT_tq$!&P1ek7%N(w_GA-MbNSZIqjqmX>F$ryFM}uG6!)kl^f!oq6=WADIpS2)40> zyk=5Ig)B4%ICU=~X;{t7g?rr?nQU{6jNo+Ci~ba4qOCw-_#+#vTU0#v-z!tnJVAF% zpvWSfl+I|!kQMz)yAjxCuCxx|x@L%8=?AQ4&b)R7g?ww_zdGI==18M`((JA8(#K96 z2HO_RJQ3*#jdR4g*T3twxbr>&VH4H?`X}cfVdgUb3O?O)rLi6bd>3(StGtF~SRRci zO`dAgcS+gpri78A`FG{!<*6X8T#tCIGcfZo={+f;xIgPT-H=B~d#Ik9dHVr+|Jo#J zt~ot1xtcfmuu*X=d`8HL$wT_UEH12|{VPAG2o|-wx)8v1@(5`0eg%svh4ebo?@`}g zo0b4e!TluRwn51egdh+$sAB6dlbYat6NqMj@>SgyXu!|- zM0c&{d)`e;5q{>&kYa@Gw2dI0>PVOtB1m3l-Ke98yH^BB0 zlcFTI;oewP#QY|*TY+O=rG4&A^yS~+*i>(W>idXh(acIAQ)D{4a@jKokei|DAR4gb z3S!JfI5PnADz8GqN#lV5|6&)CBTQF>Al~~%{#Jx`OP;1-D+JUhSDg)!Us~2@W7V94z z_B68rY$$M(l2I3-#}8PvU@sORhpu`fqDq0j?0tVdN`hv1E(K#49x!-nvk4j8)?~C; zVA+qY@tvn}_{fplN1@r`hvQNxY;=slvn;Dnd26zItn|=KaLCV-J(RKLFi>`D!e`GB zt_&7fY^E9^E_$$Ab^z)rvi}u_KcrK`&j(9OO`rT{xBmy-nNY>rO(~+7Ged6)?}QKE zWFz@nmHUhNT*uMDwcx}|Di+hM1W4O-D&A{M2uc+6^e+NZ}9SL8ftk%>GaP}h(g zzUKz3gdabYeX`1jTX#e=}0GomcV?z;V znBC0^=bJt&J)=*v`J1bgunmjIqi44rv!LtXEqMozFCL_x4BGqUNq_WJ#`gndP^J<} z9Ys~k0d;$CqBMe0z~R6Qh+67fN(icdwC7tcSW^P^!wBKAWd*M@O0_09r)@bKNouEU zCF0*tSG$}>&CLg-Dj0(;vSoUSmlin=(*sbGyS^_qT}+cJhYWiY5hyJwcx3A%;f~Z>o6?XACF`r4d{i;=;ob#tXQgF->mp2Q_R;tOHYITF27%V1GbH7 zrTeB5?N~!y;I*i!Ul$b|5wV#trwEV*&rW^6hgxw@NyU<@b9#?X#~3CxQjz&zbCW*5 za`hPTD=UtWsJjn}Kr5!%!S^9%t(W-kD3KN=l1?0%H4_7y*qtRQC|lC_6ae+8P1aPR zq0Y!unj3vMo#siwlY%XMgt*{+zAmUQY!x{ZX`}fmVe~mUqu2(|^x>8$si)YAF0bCM z%;DC($GrPHW}f-Swx>i*p6UF=`7@a76!)39=Nm|2Fv&KdzS*8*JiL1BH}WR3271hw z0tCT}Iw=TBJ>H6TG-+VW+KdmxczjyTQDmah5T)w$MZM4zeUSalfCjel} zoCM8LwEg&Tf_fW7O03c-+4n6o?ex6-q>9NHs6H42vMM%jW<|~4-HXPlbC;*ZI{?$r z@_|;5W{5q409F)GwvP^`=Ep9h0o@c;RaJMj6Dnfz z_{KMfoGJe#!QCOvQ4=np(W_AvJ1saKs@ zF_<=>#6xOC^bQ}OMfOgrn4Sex(=erojIs%?Yat*C{40LCLU~~CG*eMi`4w~lNbb}D zJ#0eK5m;GSY1I`exF$)L;~CpGW|{{`t|>t zid2&CpJuLnp?{_cE$;i)`HF~qYSDdY%vL6^-QL79154S32Ulm#8MT1+>vINHAz;b3 z|Dz=BHKd4-{f}%!;IN39`lG3iQp7CElAMl`RQ`}sDSVxUunw4=IWhCW0Jywn7bVCL8*Fs_4m^f1_OWVwqhCkzGOx7S+YbqcjtoqgR z&)ZVFmCWLS!jVX)0g&DTaIFo=92XgPXnYl^OI_k43uBAbZO>=E^!x2pR*jU6H>J_q zZqz+XLqlUIab7;LXZ*~Ey7Acby-wAQl0y{Whn4@%FbAkTop*@ZY3d+@_!glfr;h*3 ztgn`cN?Va%fJt41IM;*>!X_yC1am*EK)wI<78;A_gs~496FEc?UE+QMar)|W;t>{Zv&_w3n(BZnyIrAk8!3?a#hG@9O#_+(8T zGMp4Kumnzfl))(u)9P4w5=aQrxIbz9x;7BDhk;>_F)_btlA9JH+Dz?PyPLt14L7^YMuaGRV0y%R;+76`qrN{lR#0| z+UUw**Hp3440f_VoDuSy!^|VDL5^C8P8qBENKyM(BHb{?UW}SGEYsI}WllOa7ra3a zd^2gvCwp(MziznJU%c1X5OvP$;>XezTtV3S>#D>4=`ZC!wf?bZzBH4;e?)-KE!;En zgXi);GhcMhSs0FetVc{E%{0l%0R<)Do%FxbX~t}u$7}uTnL>m~f-V5XOCj*qoP#rGqJ3UOLm${dFiSkx?sQS^KQmTNk|HTl z5jOSVM$FFQS-<{5^4{r-P#Ga}vVax@;{7J*C5Hk)y6N@P<ga zCZ;LP<@1FOXKuykaH9Y{TNOu}7FPbt*lalc{pS`AGIoYsf6QF$+E<84P^L8|&Md~# zzP}}tmQ8;`I(2}0WLX#hADse>;by`Wq~J-3qyT7BhG)~9-~Y+4 zL*T^DC5%K*3=H@himP2Fmo8r1x`zfjVz)^J_ha;<2%(>X{@K3Oub%y7`u^5+Hj=yH zZ(rYrKeGoq zQ{^2Qs-!PoLY?q^o?|^eq?X}*JpyY|K@&Jz$J!Bx5B%Tf%;YRNQr@BF3dCI&5Y-+5 z5K%WaBtN$({hnTDvuy}jQ|baY3-*5FNNieLn;D7CMkaQ;ndTG@g2w1(OX~QY@@OHeRZD&IWdWjk0 zo>T!^zcY`akU6+nyd_XuE|#Z^1{on(}#DV%Q|t4T6?j6R=jX8U(- zw=94A`NY@PPUB-;z3vte*F5#=ojbr|fj+zGLQDlU^XV{^bQSeUDAoDBmRl zRY^v>!zA6GS0?C{@NnzZe*ccDWQXh?q%cKIwJEOhUgvqY)tJTy|5$K}-9-ftVXtzi zAf#W&0)^FXvMm0MN%2u!9fr4J8tfdZeP^f7Ps_j5=P08lD2cM3EU~K0-FQirpZZ>R z{<%YU5fr(&RfrlO$1IhroKiO4P36y@%@g=#xIThVfuBdi?|q55|8%&vFSVt;qwmSF z#M25v*?NFXNiCqwK0P;mb%BO`D%bWBrw;EN^oJ7}@{f(g>Da0v`-|JN!E>eez7wVG z$>;y#1%r4bBqb#u67A;H(aZO87S|5;St_;}@Z8(ei#(gqibqj(9n0UOH<+SV?;S-j zfefMj4}vu!Tg8Gy@8jnB&9l*oDLOc5=%LX!o%L`TCq<=}jMJr?0hbOSpmR#&P1NprvX@R0oYMdVc z({%`6KYbSw6Aw|pDz*A+Ysfy-bEHJ#zI8m=&Z$UN z_V2KU*@)do|HIwGG39>M+ph4gw;_^|uw8RTr}NIye=LRq6fhGi_5|H}@gp~1xuK#l zUPP1mA$<7ri36>7R^f)hKEU?yhej`U#3`#qN*bXb(v$;J@F20y zds<;xe+6sdmtjMnC>OX$Ha*JY2qHn5XuM$nA4QrQW5>l*s}Ut1zgL*egK%Rvx=QRJ z_MH9S(RlU#q>8{^wX6RXap>nrJNRzUD9AMeAh0p+;jcCz`(Z8-X9{0xp)Ep{j%ff} zuq>MU6|&)d^-g2Fxrh&Dg?t$4*;`1L7nc5Unk-`40m^2Y^j}Gn#KJ+K-?+7z<${ z6*_}8dR-#MXV(`v(mz0mR6-lswolJqM6imn1H(6!z-LzTbFONCnh%GIBBtYJtkVN8)=tk3Dx-e3+^MiDo8;VcPwEwK#aQJCSS{cwCgT!wms1iKj2vyBgCk|b#hpI0)Shv)^TFNWUT znL=0EqtTvZ^G4Wki=gUZ#u`w%!%p1T=6NfxbAOq_Q3y$<^v6>-@Y9|~B*`SG7o3~B zkhrS+*v>A|{vaVK5X$wiJQOz|r8W`{A4toH@k~snDfUwY!zIM^sxi%N$BF|;LV!2l z1UKkVXpqBCJrhy-=iVY1X~bfa7#X&vAvk{MA6Y5okx2jrUM{=Wb64bJS<2DUoBm;h zo`dSz%7ymK0Q81VqQQRTfyW~9a&wh16wJU${#RYF*oOEs4U?=1_qrUz$uYJSp?gD~ zKPRL0rs<(Ty;(}dqb*nPV7e;k#^OF=M9w1#P(dxWmF#Wk{dzj4QTcsz0h^7U-}t&0 zucevn_I^LYMH%>OGd9(8P`^ue4V8={-r$@4CNnqAmV41Z0_NdC{ch+k{1H2kkwXx> zvVX874cTDk3#H3U!|F7`TptO#nq6G67l~A`SO2sR+;`4|7kA#Kbi(#?CalaWo7kho8vCFKs0cl zbe=u|EoF|hU=hQa*w?``VV(XU1?lp-pJCHk3MDs7RC@Xw|D%tf=1DtRDmG948YTwU z)Sb4*uV#!-s5i`%e#)Uen*VgO)5KR$VN#wRoz*kd3D;-30N9vaXq(|iDZ6F=Dz+;p z6_iu=!2l;4{eiBAhGd?fz+m<7Oj#X}Khpb6#hfFD58Ewr;lF)bQe8uXG3nS`oxddD ze^p_q>W88A`jQ<1E`zKgoF2dTAbng<>Ni;2<#+i_|8}a$g^|fb9ENKk(IfLQOJV?^ zMTJ|RmHCgpenv~?ew2aA0Jo1KC`R3yq0gn8W^tipD@UILf~O6?`OH*h%AkC)PscUj zaNaKIGgky1J4P5)Vr(+051#qL*Xz4X_}Bn;hDUt_K7h>I@1BT4yv<_4Qic%*f;5LQJ5W zFD+w=z#|ObuVKW^;tpc{0GQDky!}x?*)~PL4H_VU8Tl-G6s~{e$tF_C6~w%a(1jIa z)Aua>+j$Ye*R;C0x<)IjUv$nN%*wS*?NE$U(Rq4 z)Y{@hHzAR~#CfJ8zdI#}IJV02!JV9en7o;M7Xef4=dJpDLnnd?gC zNq1t$a4S)zS&oCh^uy4R=q!Hd746PH^>lCgtGf)t`9M0FAYR!AVhtvT2k#|JpM_#L z&KRJ%TEZnYqYLW<92|GuaVC-;9upV=(@2)QsH_U5CHh`ud{4{_*mKJUklX!8%8)w8 zKK{QFKj)HWl0f>~af~#wFY00aP~v+++MZq(=|LG0fw^8$cNhAV*!cwnLjx7KP3aTR z1woYky~W!~ev6lQdCl?fLQYC_di>Z3sOLq}^ed1Hx_tRE&E6|dZqyGr_hskLFFhw{ zKgc@n+!}NJ9t;|eJxf4g8Gfh`KfW2T@XiXd_`=s_JfM08*_*TZfh&zrJmkfZf*IwK zx7#=Ymj9QI^v$B8qIvb);VIMKrkxYyeMS89SZPfyUtXNPjyl}OZ5((G8ia1MonJkD zw_K;?ttmNs3pZ@hw1-csqyQ?K_gPD>%!KM*@3_UZNta{5;;4Mc=(*6mjWl!vLD=EH zU14Uzye`aFyl#&@%3bGd-dyYRvb?EfiBC>@D&dLI#B2a3^NmoC+Gl&9oF)M=Mk7-4 zlgk+*6ok*q=Duf;p8W$>(d~8}3AQ@W|71*ZNI~>gX5FyMu$NrMLs`D~hR)R+3?J?~ z@sW|>Vg=_@Pv^ow;&utyeJ1eAiNKSKb)L-=+6be;_+LmC?tu^BRq)~6lMG{-nv%c5 zF^#{nF^6oAUmv$_05^tlb^tDSYcV0Qh8=Tp2WAvziOq=krT+A5r;E)gR6nP#J`EZB zdeq9jv>Ra^b}XEuge`BrJDj?(uxQzru7Kr6&s)xvd5S2}N52;@8Kgu{ee)7D$ri6` ze;$yu@Ij8DbK9R&UmR@|~J zgq8N@vRb#}{{3a0IK{@V=HE^#M!k`j`LDmRvakp=voN0zR~UX!^>(Q&%gder2z2c} z9c34rt3N&xP_^EGbS%mFUv{|mgm3)x6p9$&W8`ww_2LzOMz|LxG$gs_x3gM*TT8I6 zbAfihCelvxb?d5iapvju@m~|(&b2hEEI>ymdXINBv!6-u)}>+cv60+I3kAahMZF9hiBspqaEq3Be^%I#JqP$+M zcV!NUk9y`MbS;yqkmMF=TO~7`IJ|B$=Fwzg++qr&s_RF+Ii+q!w4ZP^%J2)>QD7g# zqD!MFzgT{T#;iKomFar)$&4R%4Rep^=h-GqbUBMn$qMcS9C*^3oN{Pzh4XXWnna_t z$FCwceAxmiA-{`ASWnv>Cf-F08oT!EM`Bmih!5t>Q}2qE*(7WC*0Co}))7fnjwb36 z2fmgB+uXpTW0mM)WHfC%pYdXEOZ!yvPLnxQ?N!s~{O82nLBUqP1x%E5SzX_e-uKQg za7lgRQjMvT1_-R#@sLqvd2vyViFH9qpKPk>Y|mlA$(u1qZFF|BOcuV|?jSKa^j`0Z5|ae< zu2Chww#=GC@f(B>7BmdJSKk!J_xJkO|GaQmytwXkVbyjqlNxT<$+L!fOUUqRZ?&yy z6Em6`z?$?}`PQaNBzjsnAC9!DFl}|b4f_c8$9iJ(TPRBNj85EYnf6PvH@5H$)wDhD zJ!tE9f2xGZXHh%GcGBT#X?ORM#-ak2oQDgNFRYg;VdVDfKOC2Czp1X<(A6VgaN}uZ zxohKo{DQz)EgMz{9!QDdoJ%nXxjSTu@q5KH{um4YGwNO1qkCf4)H7i{Mqc*BL!%Q3 zlFG+5Di1$ACi$*de`?@ej4HUCc*1+j@DHYO;}FMMfy6P3Q<6b{hpb}hJ%vvU&Z=}* z=S;rw)67i)XSqdoZ}JK|B6DLO9sYS& z3R+>-R8PEEEteqaHw+&ma?rkAC4+MW=g&w?@7JEX(>qN388K6rCjW5f&ej>BEmgh* zul(=9)j>w;-{b%P19=crk3^A^OtoXVFB_&tt>(j!`f?0R>FkHb%ik1lj-yszq#F@N=0b%J=WJ1BU1+p>+vbnsj zJ{5*92{|-XQ@~x``B_%ze@oTlZL8bgUM1S$vwx$NtYtE6IsX-`Xfko7>NJADH-f5L zL^SdMTWt9uKjYVmmV1W0L^$gh5^yJVT@yGY6;$+*RB2GKaHlTP*3qQMm%vut#zHBY zVTE^#e#h{&Wk&bgr=AeQqs4e+X$iWYPB$4veVJ_r?X)Shg3iy=A^M0yFZBkKI&2zY z@BR`@q!cM_9~z`>1k1q1W|z<12-y%l9BeJlU;~mII9k5hGoL;9zWH3LsT|R9G)T>r z$7}XzkNNz`Uw@%6JjC-ta{cX<$JEY92M)ly5`1Q5}|l!sZmvy)k*x^JbyY#~}_07b}qkcOze%?y{L6guZ?6&^+uO z`{~?)ANp&(gaubjrM?FvbUm8Pde^CA`D1Nd#x)Fh`0`1JeZ8O!sg2-NzUx@2rEnoJ zz0f_i3)$`QhMSO9_c}^-w$M>KqW-Q3<%jksYm5j?-lHb};FxA@LT0JsV#cml5nQ>a zW;LPTB0YJRnip}8YYf&_FX(Y6{rV{&!ADO+_Msi24c>1Qn@@gA0dJ!j&#aY8r6_Ih zV0qJizR$z`iT1)?%fC-RZcW7f+P-ys4qsqzu)F+IV+BDRNXwqt$N0Sc844_)1R^k{aI<+>o?aGHHVAm))UD0i8RnJ_fW% zys1Dfht?WVLvY$;%$we{PJfgP8tmio0JK2Vd|0*XxDT`tr^2PTM?eZBAanmY`P z^aedyh2#u#$1rRdGXHZ+*3ME|pszW$vBQ_JS!^@Qh0^xN_(fS=@U#50LMSY+iqpxOpI0PG6K#T4d|ZKi z(r#-Hqv2{p36Q32_XemPKcdxW#|pK>4w>7%oMdJAgWmgO8rL|BMHb8~;u}mhPwd>F zwKjNzUN#p8pY^hv3j!dk@PWyhZR+E7yVb%)9-)m~O=b8w=TIL--Mq-VZON*YZ!Xx& z{5)w-Uo}*|$Frv)($O%vU7S~H@(D?EYdtoL*GxRt;r8Wj?T0j(i*66E@-OjwZ({!W z*(p;fIHF)O9|e9{G`kZMJ*^AF`BA3TsSyG;r+adq{(Nfg*jtLmw&A5C{rtg&jlcY5 zo*h@BzrOkR&TeXj|s=(bQ*AvL}G7C`jQ3Lmr~sgBQIM1E}D|kUAgruIYbr^+dx}ZFSL@SDt*7 z^yU!3fiB+Gk-yjS?fCLQjo)aqZG4J@(fMj?hOw}6wTt|I(VFz{Z){z6%duMjV&qH2 z>uxZjb!It;;J0>-W14%hpZ>hn4N)@+t(;L9(t}1V4~?_Lh~a-D1Qd_#8F&BMa&SSk z{WtU`nk6eekjld;IEka6fFi!fIZ7-pekFjxHhtr@pz+T6B^v`8>UQoaX_Z_*`M{B8 zM7IWAk65f7!4}fqZV>U2k0e-{Y?Z}%H=J8F6%#Rsh+XX(_8Os86Y{`~Pmo;-mjMN4 z1C+j+kn=aC34w2|UJt?4Nw|V~g9K}_%=?YW;RO=k8>*dJvph_a6{uBh5+tEpd!UJ# z;x?X6(Plj|b*FFF_0~*hPLQ4&1W`{tuP2%2c3sLT-f2?MvmGhr<91CiJsPXtLyC@O zXpkj1nOsS3^vZoQaoX_aot+&lGIt|>&7N9788$60(_kMm{N0^#|BciHXLqE2ZNd*D z*V-nZ!RMPdAOCS=EvvT(JY#{!*Cx1BO( zXMIyS-+sKM_N6U7yY12GLd6AX=F=)JgMrCmeygfDZ;oQ;&bJhGNTs6C|Ff!x?}r-4 zsTXtM7OzC1Zrp)id{Vua{^xbyck)lsi%Xs$XbL=6ubGvVjN z#9!eE!`8{`sN`^P2gl!ddcyH`($wlNU3&zzg=Xqh3DD<+isPl@9;xZul zk1uUIlgfslPt7qZSEhB-A6&D^;36|OiCO=Zu$MOZBu(A0^rKhg30Rbyz2lxOYvE)- zzb(&eOUE^)3((d~!j|mU!0~1}xnG#3%^^dDa z{gsNRbCLUMl?~!(POmN3m>M=T8JgNZn+nyEll(pIv6GKa9n$|F{lhsS7ZwvNI9e;#Xwto&Sf?=FpsKbqZrnWY^36L zrY{;q0sr@~^uZwedjw3@P5;mSpe(1wYTfEHeF}zMTN1j-Q4+ z$A|gGkO+-cp}sg<3*~)umKM|0=lvd?`(;QedcyjL9-ZV){OBV1`RPeZs0SOw+Q&fj zS$15R^}^)rmj$nQdmy2|06)IInz>Bq#>UR}2W_?Bb;pD}^XN*zvyPj^3y(w5Gv-KV zw%gPIZQPo$KJJEhycc@QyH{ZB!+-vh@MzMwZ}Jwh%cCs3T;D+T$eah+HU=mETe9(5 zyWwRvQQ=d?^Z8cZq5M{CpLbQfH~-|@r3zl?l@rQ2wS8XZBZ?AD)E>kgO~!iUk-X6l zi9H1;lXR^3>L)-}qky+JLOZh%gmIJ5JT5(JrYq*9o^*as@f&>b*$agGv^avjki?MMgILl#T6QwD0$|%WpIS`aiyo16+zW9MOM=>;QR}B)mJ_l4_(|$VlYhCKCOCinK=_bG<-D>p-r4QXhD<0{Ay^bKLP)m+lq>_hi68f<+a5XN_Z|ty z1uSmfSfqvE@wAc6y}zsxyVJiPa7CPDl{rpN{M%&VT)e6FVA=i$i-KphfQ(`Uy?WdN z9V?`JCkN?up&Y&DA^`QDvlG-0WV>bjIjD7=)Fivmg=~GU39nd-xrGidQ?xNgLB@PBTh%7m_u~N5$8S1|+#MBqd8IZPq|$@_Grb{V z8JI{vT;e&-0%g^8?tO<*Rx*OHG5-lF1gnSN=8{}alW*~Mf6yFo0sSbIYk9~{ilja$ zMfH+`0{W()(WAri2eA2`{632`@czg&mvH zA%=+&*9dx`A>QMaC{lIYscJp4=)}2>?4(4+uIuz$I|-~5xD&j(Q?h!BIZ3%V!QvO^v4VZe1M2ZPmG5bhq&zc&o|7L?)i2;c=hF# z*u+p?&C+&jB3Iq_K>n22iJhg+tlVLBU>C^PurbQdIV(z0RV~1&^Yf>2 z!}ymy$fA$l7}Kni(Yk@z^inNA6?ED{Nm1ISJsm!f?0n?7K%6gY$^MjHWZ!Y|{!-bC zePqYkH6@yV?2u;qV3=>r#2q>%=|*%u`+jvyVNQ9m_e4Xo$dexWEA|%5I&}JrSBs<^ zX}igm4ugs=?ybd}+e=Js9k9uc9CT7wiLIXn690=e$ytL2FG_R|S}$XFZo0Or zUV&^GW&t&{FRryL3&4Ss74$xH$9K=-|BMaET7RN5Gkql5tRS!q5!rg8vnJUj6|H@8 z2fyD7QZLlp8Ltkj#6QbDT&THf&0;Sk3-AQ39M82~LP{>b{dlD>X}$WhsEUOY5~<*> zch}T$PGC>yjx_rZqCOKA(pw$TA9_#%lD6}5k$IlW(?lH%JC;^$qx>RCq2*AVw*mCJ zx#K%{T!cnlIfdC%+k?3#F@k`)%`oU7R=VES%iBJrSi}((o5vu1Cn}Zoi`aIRMsx4m zw7w=*g7iyo>eJCVjWt2AD}AlVvfPd(yS`E?r(*QWV4_fUOx}I=R{6_%ahk8AUk%tM zij^_zPzT@PF&-BX8T@#J@sf?*?yk%yAw!7fJ7G+7>~6b`M_TkS%`Ron>@{0VuzqkCh6bp)Ho@5Vu+vg_Y& z2F~KYmHA6xzgMB7)4s&bYVxfjLXcjmXx&}*lTq^bA!2_b_;dooVVe$rjJ3^LdcW=Y zx8Jw*`mpz%^oW9O-bA8gKIF?eqAjJ+zFP|Um7D^J0GQAYU3O*s`P>I z=Ew38Yl<~h$31vg$IxBLi~H;aVkTg0elsfh?udNqH=WAP?$MEtdyy->fv4(W%bk7O z(zV1i^ypG%ad7gKzLRJZJYxzQ&II>d=Z!aEt@4F;H-z)+6X=8M)LFPAifmA9D&I|dU&0{-q)CDj@>XLJI-#6xZX}jN%fWrSf<4x9O&CO0^#{qDcX{iTl zx!f_6oEQ0(bLWY%ov8uQ;-eLQLb;f5J=iWS^ zmb-hoB;B{!@%oDcW_|3{;j}2P?E97QP=GyFrT+f9dc9%jgI`nfq$XeH!+)E?0nd2Tc)bT#yL5YTCTsrw;lVg7oRlw(nqFbs-7}S5~ZHD6C(nMY=ftdk<+K# zb28w1v%@@(P#MifR>hC-*ZNnUe<(RIX1}momkMoTRdcd9tn)pT%)dnI#v|cv$^7hH zFuXau!SzqX16z^J(XR2?`89KnO9Dm84Hx3(gyd^e!SR};FMErML#iJ@=2!-}^G0JD zluvcrXU|*YOWLA4T=}<$3XS^p#h4HZ*5T{f`Fnw7JPFpPc1UTgvd>+ncfdId?IL%v zdaXB}K)j8{X?B6W(C-Hvz;ks1a^j03+m+8=531Y-oa=F47slA47T4Qb;Ky$9^o!Gz z{xeXm<#(^9z1JvuMwXnPnSCEF5v!U!9Vv&7+N#1{3;-4hOjliFpuJsh9gVYBY%*Y# zfM%^a_M>yl?M?5$KVvJC>pDWHVzSvO8iDal{wS6r#OJfC4}Z zsm-R2V;faM1s`=OC&Y`0L}(*||%# zKYa3#LsLx9b~MiVoAu(fQ^;I`72mH)4jOBRNm@a`$O`F3jS~YH0R~xjTx2Xow(or~ z=vBL9wZEpaZQ2@QDF2h7Mvs#uH8Uq3L%T00@wGKM(>?l>COan<%Ip#Thqm(%r*8d; zhCJpc;rczRloAU|k45|W8o+kqo1?rkX;NKY6@S#kfTK-3H)L%*>+7&8Z|QGZ2b51O zd+Gl&DNZ2fi`KoTYfGNEI#Hi6;v!-q_n95F*}+`Xwm8JoS~;oGQN2uUW2E01&8FCj zK=s=RardFhGYfNhSCn2a8(hCvYk5Jcu-wuce(8XXli1fSUY;@>UuhX8Qr_CyP%F5FBP;BIO^;aaPNAwaV-sh&<+L}sb?f{Bb>2GmzI$DJITNEMy-@b% zk`w#GqPK?l*#qrQcaiO2ipd#;vpKWl`8JNKP0=Co2DhuRp8%}MG*xV4Ml2yPdsH)E zFYb8Z{t)^Fz8xD=q4F<1lx(Qlp~YaF_)}9N%J&!2uJ@1Cu#gvK3v9bk-s~*#^%GyJ z{;AAOQDX8POS{g;EpZ0Gl^9W~9Ds~O>N1Cfm}VZ#|lsf!A)`e52fQV@$}9 zNu_$vvO}w{4~+X-4BT)-Dba|`Lsk5UvBy?Q%j{ZZ_GXz~5% z49E9~k`cYDCiA;OvF2dyXoUCM6S2}9&N{D`DzKRzRoAL9cG{o*z3fwbVRYdLjF332 zpS<5uhFT%Kksnpi^H1ExeUym<^*vd<92rqBk7mENdU$*#-V#u*YmY@XU-t+w`pdDg1Q!8VoyPIhKbO>}gHfH`OC+dO~778i*2iMATq?FZAKf zZ9}Eo(C|C#)PQ~v@f>$LE8uTd`DHiG>Rw8?_2BBWk+y`y;)atsi+5a&_qgFN^XQ=$GZ<1E+16UB(z1*)A zcj&}d9hH3d0NK=F)t0#L*V@9!rXy?U%cKhse|~o{G+pyll4I7t~cwV;D1P!;&P` zGO9p%|C2z?M~JT8=x(rUpgxnrEioo+R-7{O10cW^qE)=9Q^w^b#!fMK^Ulw_8>zo$ zZNk@tGk>DbDkkq1*L=f{a`_@1eb@P)5fQrIuy8#Y^xb#z-F%kpiE-DP!BtIpEE?jE z%Irz6Q74(Mw*7_?wR>68uqJ@F;cYuxT;7&WHzKx( zi)z|4(NL#=4g)_JG+a~kX*qkhE%IkAV2DpW#q)H6+NH9R=Z8qbY0&vMLeiB1eN`um z%zbBXY;2ED(pIjAz`FMO{lP2k=s`a$ysbW>=hSkwk8J}L>bp1P$SL&Rcqnx@+w}qF zxoho>xOa!PUn84Xt)C_OSe8NX>qHd_=3huvNWp9hrKgh>_8Scijm9huwcD$*%{{2} zcdwR))&SP>kG-2rl#@m`EWWa1$DoKv_L#QHZXp z>v+DTS=Uu(=QWs&WsyX`NF&5XrDQRet$5b$>lqm45?b|XatGsiQkPy2bru-79BSdR zdX>i9sLC_ZmAXBtjvMbAj0|I{f7~1Of7SLT-ca{%{P0K-g@npjQ`x3XS+cK#lzkAg zl!ReYLiUubrG+RvqhV%jl`XOqA}ae>Msa5eBiZ-ucYVIk?>y)E2cFL9bl;s*F*Bd{ z`?_AM(IuUP#^FTUTWrubL!rs77iN^+zzMLgi%%k*J?#)Z8z@eyjLNNE7|S#WBaM*^ zDsmV7Q1T)>9+s8Th`5VHGe@(L>H>fgC?I4lo|i7Bn!jAubUOiI4vDS~e(31Y*pAZC zt5Qfw?~At(Y~b$X>ZGCg_ig>(4e*#27@4KzthB3+xSc6^^z*)@v>HuD$$^jz=Sw7< z5J!K))5uR3L(S`sm4K44%z6#7CgPpZ=mcxe$Ez-tEk59)=2?3D6l3?qtW)2M-NEx1 z#PG@9N&7b`@Jhj>uf5Zn@-J$|^ktv(&G=CM@DR{tGV}*&!!KJdyu8$LzKv~ULUGc$&1Q=+1t_UG6r1KkXL4S80b!w zxcAf+!(cl@2wahjRQuVXbGz2H(x`tYZL$M~|KRf!)!MjriEDmKTAUi_zn-&OI-+6I%L zkboso{b4ir;Qx^5gZYxMJ&lo%%l^oxSwQ{^gw*lQA^$bCktgRpC}gpZ<`*p#q*}CQ z_=aY$>{zB{)5Ix^-B>NM+SkALnBTg+dW;c5OvG*&6<+KlcXCXA4}-mcVYbd+&~4V8 zo!c0r=YubO%|Hj&xRY`_wGoW!P7y#hr;%*(Q+zADNzw~3tvypV5$UOx=7>+|QZvcb zb6lVRgTIt=2+zL`#PQnDZb_7lzl`H26y}wIZ3=&|uX`!kGFS-q1WP@#XO_OZ-IQ~u zm6v4Yx%6|50|x+^EnBh!r0I25s&avrB7_v?3*7J}>8Wd%?NU9jP z)a8(HEmpbg8sV2H`HR=`xdJlwc8F?LoyE^QUqz9uWa0QC`W%?&-;VoCe3{23%aqRM$UD8Bs^0tQxd*w7J-d09D z2hqn(>NS{S?;!cHB$5f(vWw593!GO>z|QZE8XxMm7sc9V4F!HPoJz9GT29^}gki{RYOd%sT@k@OLmtD`W)|8Y!`hf zF1C68N_-0YOB0PSabrF#Jdwszcz z6t~3PZ&=~)cyt+zbcFw8i_79XD^uft9ALU3?2|;bD_pLzj9T+2&i@fWO)c23tp%HD zt<470C}O!=K@QebrppQLTqRxr?!33V3-H@b$#2`~=vgu}YZCkf+ZGD=ifks)L(eX6 z9V~U1y?AVUvx4Bs^fe9V6D7_qs|a;V`cr{q;o0|$e#xq*CYd_(?B*Qy_+8o<$K@zQ=WQXEROf<6Zfp z?X6Vf89t?i#Y)SxCT0^0bvM4KM#cH!U8niPHB3%YOek+HAn$taciyfy5p@6{cosos z(CLmVQC24zB@=QS`VFiY*}cT6?rTw8S`(Exq34hq+sEQ$V*~A8UrVo5wsvVmvM8c> zPo8PoelV%tbWCz2jU{+u>O;$hwM$yU2l?#>d0+7&lC#dkL=rK*#Yl)Jmx-NrG^TYq z;t^vH0Zko2zxIHh4J^*>oMjki%%Fi0mdWv($$$n$I@luTKA52g&}+>YWmUsRxoww) z1#z}oJfbmCxX~L`oM^S64_HTO4tvXv7cn(vEz;5V9`Vj*MIR;&n+0l{S~Q2fL9y%J zl#-BBr22-ogaC<_uh!0zIp^M8YhrCxJr9(Cr#BaeZ-Cv1CfdG-uaq*PBoXpcf#2e# zv&yHsfwNTek~-YW(n4wHETYy0 z(n)ni7jwK75C*6lg5v~c7nQJt=c9af?7Pf0$l^E7r_3&jp4t=lh4n1W#T4~=P|qRx zrv*MA$6Tt`TGzn({Cu+$ck^(@4|a0*>cd_6wxxLIktgh*7N_L=`!h9}F-`1$E@Pb% z)A3ycB6HiZoow${ByCdv-Kj_}vlBzCb>-!OI*lv2=HO@322u*4DQ3q`d5TQJTe{M;R5!tRnL;lRGeMgVYp zh82C%JlV+{Ev0j>UP7dp{2qX5aDGrSSBG@#%-n;=e4sI>M#kiQeu!u0g>%~Dp*Y^S`R~#@5C_?$l->%$egnvsg(L4h3 zQ9NCFZH4u%Kl#^fpaJ?`$?vsDJ9oWd@4(5c)3Jc~b=C}b%acJN_!?3l2&J)fsIrZltMrz|Ws0NvkIxk<2aajkPxFD4rtw#YFVgmHGz3>7EJ1o5Sim0LpYA7 zFO8sK+cE6FH9Hpr%Y~G-5LxXT0woY$0dw5esDKd3`=@!vM^4Vw@P&R~yV3w37CtaX zfBByr@pBk!BruTujhewDxBKBO)&gvi@Z=h-m>IqTkYazk+~7zsB|Ls=Z8>dQt^TC_M7M-RA*PK4C}O0ry4X}@C`sT+Sv zdK1lSQLn*q*g|XFAE2D?^Y^?I6>c|W$%pF1u_eUf|Y3apD884*bQK+7=Xu(-!(gDkLozCaUYx@M(F z3k}_MT9zz|cbTY3h7Ut{?Xz;_Zut(4uwA*JT+8tD+;b;t(4099?-IqL2sqOjff5)X zL_FGMw>QW_ME9i`Knr9#5@U8k0!|I?5PvX@J~Q?ZUvZG_JG`@hWnI_j0i6v}9SMG* z!1Xx0K*>#~Pt>4279I0PYhu<9vnQGazhUYE$o06rd3DyO?_j6wYslYKsuRq;WQA-| zOXgtkQjp$ZM|I7ZfUj^vu|hpZEsm8s>F;_E)!)lCsP{F$ip+Hz1lP8;8axM;jC+4V zkbF<1$$<9KBiDyo)2LX5ZFqjOz!Mt1GPjZE;eJkQo#Y=73P|8*1bE;o8W5#S5*JH749>e=viLR+peF8ptO(|pgg#ECUq{59G@+D zDlPW`F5}u#j4lV;oK`;5e(b%x_c40ud16S01YgkBUsF*XZ|435^6rEM2&9Pw4zfv{ zSFVHH{P=?5L#iND5)af2_XheeM4Mo>zqmP?ieG;HJnL$cz)H+dULSPOa~+?0WHM-y zTRrgXdb=Z;c-kjqv)<0rWXmd!sgy8H z)cEK3@h9uCI0@F&r$|+SrM#RY`9T!I3bF{4(rvd6?ripsR}rbS-rH1TZ}oz16Om)B zLtR!W_&BV}YwdVqWf5J0dbnzasA*W8vmrHs8RVP`q_%bzGpcYw&l97$L|4p8e{U@~ zj2v!R&rC>SxTgZD_#3uWHZZz>HK_%zN*U<_AilBiyb82)bYP%kn5(E6b6WCWqyheX zw*wMyvz1Rfmb}@1r!6ka1+mBe4d#>lBTLB&dD2R2aIYG?O;zX~4**E|{FcB3I=1-i z8q*CsD?eN9jEd&GFKKE`ax6*^R%ly6YV+KKb=@@0(@d}%e#B$+_8n`k#r1V6+*c8o z<&1nHcgekcP-hkR<=e+X3ypAb{mlSV$4L4K>ZQeyx%Jkaf|G|BJF5n?(aNK=T7?rZ z-Az!=x{8~{bH-dYadBW~gX&fLBj;{JVU-sdv~TDwwdgEZgZ# zCIKZmET`o{4?O$oU6XGfhPHQrNwY-uswv%wIUzYUcp@v(!%McpXf_~{lli#%b*f;`?@Vz0*Mh0pi*V5MRL!2BS8l+xyXrito_3!#0s zy8V)cIYEs9dfk$m2tN@m?iU7zBsL4rXHj%X;Zf4IQEKfo21}_SFIyqgrBVl6(Zol~ zca7$E);A_USeEA%yzIwYi|Fq$d27b8o)?56kciuyR{OXbt4nt9+QY^#Y%7e}P*?~%EP z(H7M?GJM7(LgO<90Z}L;(fdB(<#N2;Z<-BCj{_XBc8)W$NYVe}4`wfD`=ZL^?ud z#XcNP5pH^SP4xLa`pG;%OD(r=Rn6%R1R?S9&!A&;djjN8(Yj9s-2`?qmOzax!PuQJ z4VZWR(+pp_o9{{5pKEY1m)=F1;?*~9HWDC9+=DZRV|UuWEW_|6I$*geA6mby!BfkN zup^+KDmSYUN!eyf83Yh^p>(eN)i|W~bhQLP{$IW!(Kpgll=?%5>%xDb6Mu`_%9N17 zgYPf=Ic!Be^=iBbBGl5 z(vMgUtpdSV&*_dS{IiAuZlkevU|Qbag>Cu8?^Vz$xi*1Ph5k|?cI9vX`MK@Ed!}3B zgNICq8fRD?zjc223?+uIC7ag2TUJ+b&9^@g_i;EdvxS!>bTHosFot=OD}Q6I}*$0Oqog z^wXe-ZiACpUA}*a^;0cy7uB%z-ZF)9r2PF>uqt-+tb$mP=e~G?g^&o?BZxtHA#*jK z7AL!=m*B$ELzGj*L4FV~2|@CYXN&q;_A@oSQ!ZY66XCl@?C-(1>23-!*+kHUDQgSO z!SJmImIBPV9sYji-QmeM1>}xV_)UM@#$QYnJ1!eP!Al^KQUM2K$#6-%6EW7cM6z#I zEFXDs#HNfwW-VyFF!?6_;Q6$gIPGtG8<6Jsf^OiS_0t#pN#q|n_7J2VJ@M}YL@*rx zHWWRUp`o8NB_qK*mjCr4)(XRsPUIg1V5wm$#tUcN%l;S=z-`enNXP7b(?|5&W_bNcG=Y{df&2FhcrAjnoQEm%vo7L!s_LlGXTX;fQM%7F zBwYEzTT(537;**jH1Xdy;U^dwe!>I?EPCkf_iv?)eM^`UvCDZaB?cBv5Q*ltkuaHV zpqC|khbxov9x$||lWss{+u(FXL|8>Cteqt)?OSD zoC}Gu?;Uia<28IB>*UFJy5g#SWJ@p zW-!Y4Rwm)`X8yy@Qej1FDo)CF?RV4C*qBjX1FY6U2tU>VvCV(`HrEyi7FM7YB0_wQ zMB!bGRbh#e_KM~12){+(34)!QK%FX#Y@%iC*quSp`kidSgEb~;+de>CtmMJwH#bGI zhIlc|){B%AW7^{n* zSN-KoMt_Sb%KugG@+t%&62V+|B`Fxf#^nQkWPhL)Mwxq)dbw-ft0WOIG#4an0DHe_ zDjJrv{v}3Os#l9t2O^l_PFlCdTq)JMT;wjtA9~6iM0iOx%K&p@5#G@o$m^!aU-a-#Ud9idDhg@zpg_ z$oK6Xj)KfyhM%P$%C@i|tE`q0<+mexo^h8voh9=e6s*qK0iW($>Y{(m!0}_O(hDcG z(==yn9;C2N@wcw{HX}-?Ubp;X3Nf7@BmzS-+sA)ecE1@_BMbmc*W^vU-{T5jEZ(O-(3Um}Oatc>; z%Ec6NjWg1(`BX5ZOV!QBz8@Z&wZi4hJ8%4{8{BDcoTO&sL{var^czGBW%7?1!h|5I zl3v9I-xhyc{dBEDsG+eFr%kK{BzH8_DBEvW{WCp%KiG{gg26t+B^5wl`D2n~xkXI4 zX_J;!7HgSIsm)SqxXZ|!&fI?1QbRvtlaNULo>v2qMcUTDhSGKoqt>fgDDLOs_Q1&nhrCmZ$&CuKmi*(rtM=+P7XR8v zd_st(VoNjV=lSjL)a=E|Mw6hQ#}aTGTfC$@#xN9T4yP>RolHoSX3rK0jUVGFT@GJrT>I!30DcaRPgOxxV?$y*!q}!;FiQ>Zv2knd z;c5DXD;%6O9>!JIV`e^T-E1A14+}dqsO0lvN_s*E3Agi`dF4293S_k?<9U>%pY#Z$ z#Qm&E>#$P#w|j(sT6A0=VM(ib>2z zze<7NNf!cVK927kNSqsd@EMXQZJuL;o)>eZtB#f&H-dubRCxrDcV&;BQm8q?+ zPomj4w88h#&8d;HZ_%DRyd=%NnP;0PN3$|=L!LAWg9@SiHvF`$)f_t`{PRDz*&wr{ z!VOtvj~k^xoWm*5s2hI@w378Ue&d%cz{lnkbEFt zmGoqT2Uid+<5O(eRUf6?%n+NJp$o+WdKz}iKpo|0>~(2ZaD)BLpPpP9CH{K4dI{Z_-g)Fkv-lSPskI+X>p zm{sVm?8CS*Q=}Uvb`*D5(&p+5Ya80{wl{EBtZ;_!Jl#&C32Ers%ggHLMx&-nmymsV zFd*TxUs9XftpG#3D=Bh$rek*$dXm0yiRYlrJBnA~hUD-&&@M0Aq_XVmCihAoP|fiA z*_xp~M3++h-In(tOd{Q{ZuwBB#v2w%i=xSNhPNc%mWc;S+{-1u`vsFf8(h5Agp%xY z+{eKw*{8)rN}lK*&8CYfy1Dypz4^?vWgEtlv}B8G%z!kxl#?BA^0>?xmoUmD8G1oO zbr%dPY+vVx`_RTdAq+ za#r-Aj(ao-kSFbtMc{ zA%)mw3nfo}cA1iyWI){lE(y<#;F4%J={Zfb(h+@^inK?wT%E|r=jyy@j(rzvK4TYF z+Lfru-N($z`F)|7QEwxV^;k*B;BL0p-;P58@-*g{`JVG@qdDpsNcQnSn%j&AvBb|8}6#uu!N<+zu-EYDY?G(Vi|Bn5neQ(3m0ghRzWV4L57^5d$ zF{eo>e(;E#T6&^3P-=GGXofOhv|^?aK|jI%_4S22*ToLncEk%QdB3$lTa(^@k4c{a z#Z(#D6YHO6N7OX?RSZGI{PG73PuMB*X(;Ws_0tG9(~Me|!#6ds9@$j0kMvVQn;r>?269n%-ekO(%wb;;h!Wc~V!z}lb`88ghl)Q;DS*4?eC#LiC&W)_#M6^e!3e{r^+tK87Pq~gvwLXbVw zbOc6#H%Mlz8Yb>Tsw+S5c0hrmc};r1Zn%b0mu@;@92#~Z@NR=9$hm@&Akwvhc(y7h zh>wZcPnD$C5tEK8bQG0UI0QKrKp3(P=&efTk#?ho#`|mi2eyE)ib5h&Rs?~uDFJ!x z{-PmReV1*3=`g=#LSCV%g6B8+{oko00~Z4A?Ohf z>`DU%uB0ru)z5=bN)-vcOdD~Y1LI2!BwBf_1Vd%o-#(yX4i*8JHr@fUdbbp2hG4Ko zfsGs~aOY%bw*7`C_G$MLq)0TfJ;go0V#wb1xbKM!<3zxB>x~ z^nIb+rhb0$$#yzk#9xRQqPyvHKLWXuTTO*>$|4*#kYlhm7fd`I5D)6LwV&FxxG^yZ zcVt-+?4tu4DUl?X2q7QVmgrR?EuOFH}+kK$pF_vwKWywor*P5zA9cGcLRr7KOc|p#tY)(c#&qW+T_zbf!DphscyII4mef*KT$66x{c?^kCEJd>c3Yc$ zehu++e2egf>cZNihs)OGnEIcOOf6^(qTEV$0U+E3wTJFh52tDjYsrL8CA9bdimH_-gQ5_XD^hzz)7W9V=e|yth^ll1APIGO7`e z&Xf+)a$qUq_Ev)<6j*>$O-oEX!Wy5ilzCWtgG<3WZ5GyeiKkQN6}BrjKC_DGfaJuO zD9ixFxx3=7i@=3$8J|sDa)}FN2`t;^i{x4`RSISegXV z@|5(`qBo;&i-bQw=&ssAn!4(#!qyw_%M zlfXr@MOHn;#dCtsDQlW?pm-kg z)ML`GEsJ0c$Uv4PcvjzaOs%WRwla8fw$;{yn_>kHa|m8ITM3fXY*<&~Gyi#Jq`daN40}AG*{?A&k%Nr`9J>+Zw7Mr zWle-KbL!&`&Ph^#`ax7~LDpM%E$Iea)kB=he;rq7qb&&KKJBPG|KC27pRVUj^Tcqs zkRTO~BWFbe0Hyj{I#?Yza3{ff1&}u##G)F()6Z05k zz)^)@#@hI68L5R)suqUq?gwevqHHscU*R-5VgV9YwQ=O;Y?uSPlWjFglZgZOM*15B z2)+-bs(qstb~x6QLjBdcE&IVs zHEP18b|$ZHVDFgl63gS=NUs`CbRVvOvvRr*AS{B9j*4=)dS0V;P|)#h5*s)6!PHtU zi2g6{sO=^&VQUM1_e#}2UB6iA0t9Jx`Cz~|%hJ4pI}rJMB|{(67`LHj{p*#?)9%-( zi}mc8Fhg4U;q-Z}DGwg~RePK8P^`F_8GFu^7S1A8!hS5N^DAPjU{O(DpEwq%&hb$C z;b-;lQ{5D&utM~1B-CPl@gt)BOPtpYOG7`maL#$~f>qd0RQm>RxX{;B^q6$K!L?VZ zr;SA32N-@UlzW%uy>Hzg;KBE@_lPXzEKJ^;@Zr6@CXAPy@F9iHvNXX3;}*|@zv`zH zwV8-u3Ucp8uQF$&$tF$W<&O<_Xt0c1xq_tQ z*yd}PjQnC7kFCeiqP}heAYit(xB9|q_bh_X}+KA;c4(;ve-rq z_HM{B_7T8kDgSjcUc?&fa#&|eb1Q6^NY?e-I8Qd}(jYPR7HIUc%NV}DacGk=WK5}B z%D0In<_l0*@D%3IMNfCAwUhycuc_D)X(eN^8kF#d_6rwr#ZOk~KTA>-*XY!*9RJyq zNilOGFZ{mrBfdieWwyZOf>okCff&#oLqU*%a@omtdxQ^6H1P+}2rNSqu}`=YMP_T4Cd+LIQ3XHi&0)qj>Z4TVB@YJPn`k z!=p5oLM=TQ=WT`dx7p<_hb8(qO%ZPxC}gwLwnpozxLwz2l8t_0{it-;ah&m_iVt4) zcOGZlzg!n{fiYx!aQ%m?)uUEkY%5&I@c+Z_?yxq)Bw8JIN{9I44JZlAlu#9KQtjeBYHZR$LU!%73hCGY;4WT3tj7i4)T8H^WLh%h9i2CzX6M@Jnd zPgS!iv$w*1Ru63W0r>SWM5%KF1}>po7=0S4s0-eKIisH6C6&7dY&y5wHCe-v+zw#U zFvzQo)Cv1;R!$Y}LvFdW`-t~iH4jHQ2r-!Wrp4j`xPP_bCa8yJOdG%gEqExjq3NMc zIPmYarV$9wInV<-0;tP)J~afrTzKChkVc*Wlj$izXpQhy-wFX^X~|IAeZ9;N<`q$| zBH?W5TS4pwD~N=m3QU*30nLBcEA00l6J}w31MPgpS!XgFHSc6YR{Qi5eoJ?l|NRb`9 z5Q7tn&n^r{odsg!Hppti5%V6>`PZ`ySEW!Lkk@E10AC0+Qq{XzoBbWuF-9Ip^S}RF zfx{sJUW}fQUS!T2Thh3L=z(pOuLHvzJ#Uk1B%}-@R0fjAV=JQLtQUMp$Q?qKFRV{w z7hLY4T3|YIk?N2-5Ty(L&)PSrw-iZ7BYS*%8Nn7a#0VrCnJOYOl#2{)VdFz2L*vS>0M0ss~l>bWka_8)yRyp(@x*N8p` z(^MofFAYr5C$-DGjq>sfId_5U+vB(V@K?J?KxkF5unQXTF53}w^M2f|7e6}dgQgv9EnegDSTgT0P-etqU!kJp_dW?oGaE$a@*Y^$o};n31qTdZst;8;&*_%ANNu$p+( zz}&l9JuF=!!B3U!+X}`Z1^TB8d}(>L&*)MH7qiNq>Ak@F{Dc8X@E2fkhu*P^F0@@O zIB8MQfj2e&k;gY~@H?|=wI*^1^dYHUa+G-ocakMk>Q1{981=?@(XQDnO}-cm{QEAF zb~HW0VC-sMJ?>VijfeScFt4>U`>1gx)>FAu@umGGw+joED^j7uzj|h6xajCTkNHHg z`t$h64@jlt%@d^o+JL47DI4c$ARbtql1@-^@q+Oq5Bw65XK(-aY5z+5lZoLkk574B zgX8%(l$vsYo3tcBB&Nh<_Z#9#n66)a$k0>v201(wA)EkH7v1vFei${lljZ=AzS8{# zI$7#KWkleffM@Y$&Wre+jO0@?_53?__XY?GwyN_$2@MoOctTa^PK!pz8?Rw*vlwqn zorji=s|Kg5b4FqUO!`er>Ohy|;Vff>Igiv!V(O4u&=nKwrGy(Xeq4SMnXG|NHM?qos#G0=KaI@jycK@ac;ECtgLv4kdw5+`mb$?BDk; zS{Kdx*Re@Fg2FOe5e>}Si2eN62^_z^StT=uDr;MQG2O;+U zhD*Ne&~A1=tvx9y#q#k!mTUCgcEy=CzlefA(=Q2rs@=FIX=77yv(&&~W7&1jYp;%V z_O}@!>&-Rp*|)K>uJ^f2t>?*-SVtPfx>mgQR-NZ0i*2M!o{B5m`?5-#>&k^Mybdv6!sYNHgDt?w-xn~m zd)0fP1Tcji1Npqn^Uumy*bnlEV!y&w?U?c}&Aanr2RvWrJtjyGc5x`VzBQ$|Ej|z# zLj7WuqVc!8BW{!&jyPyr=~F~s@IfCzWOGEYWT|!}>0`s5@)RGF?d(ZO@V7!PxdNwGrqc7u*IWTh>@Z^40Nk{|vnTx(2 zY4Ma&A@3CmgDI`3H=#wkRpP`;Un)4pQ@#uq%?1a()u*rcu6!gwJTy&T3PM+EC?~LC`obbB42i|yS6&}TR!SQ#%+@g ztzJmIR~rkKvkrj!&spDX>}8==Q|U#@LfI#TS}H>1Q=N8Eovzf7cJZ-*Y4DUAUDoxp zBQcqC1RXb~j)o7NFLGN89e(p+k1aWE4wjcbnNAawqIqiPyct~S@G{GTpjlqV>(=3* z*6G!BQH?vfnph4zWL6Kc2=K;EGVejl;_U6lCVN{b7oI(d8LwFaB%#;Nr6>&^v*^_A z^aUH3k$91AZr`V3DrX>d{c)yQ;effZDlHxBe1%75PO~l5L_&wBknwgxm6toG88_4I zYNi*KQ&cq{qsryO^!zSe;PRM$*h6rXJjGsdcQ~71Vd&_3P^Sp9su2&4J(rBq5$Sj$ z+rb1=tO?Sp9xeGTc)7ocW&a2>huVv*XuffuA3ztLdOx5(!=*HlNe20Z{8_3~@>iN7 zs_f*$Q+-bRzwRTJGqv}8^>>tT5=c43BbxG>7`dpv2!DH8R7lH@oANQYaf@#fLosH< z;fL*`px}e28gXkgu?jk_^kh1zu+JlO&WO^GzhrUF@HFPr)i4B#v|V6n8R``xWbRh`Qudi&+93B1JD9HX5BfK(KqC8Cs_-TbssQ_ z;mDHsN4oER*75~IvA^Mn#vhdv2`yx^HrFpXL{gfWA?0Y{d{MMvZnFi6^gq|-3Gypr z!#Bt}e27E|aE1Jp=;f$#9$8c3-H+arac)96Y;+`qz~DfFziI~koV|n;#pOH@O}3))C@KtO1XR^kX? zxD*^%(ER?LMOsdi$i~NV?m?Yipf}|tX6%igU6a)PT2DWum5wfB=?PB?Gg3g;UQ5~O!`QQ(r!tWy=4%VODvBbWN~%heEg}kE}HH6kI>Wsl>7;j*9F8Z z7CHxe<`YC`vA+oP#beRUk5>)2jGadWp5uq>SOpV0hmrIgO=E0`0wgKP^DI{T#l7Pe zHIa3di)^WGDL$T?Z653enF@W1;BK?I?vo_1*ds%t8esFVvTS*@6M#VfIO|BYaHGpw zOTCAX@ZGkkubh}KsP-O>YmK|V-qdI%pi<(N_jG=GBhP6WJQBn+0zL5kQ;>S}^|>V; zB7A%|!P;D$5x=tB5gO?=JASiG1Qk92lzIM7jG;Z1?SNt2y8U&77vu3cElooo?l>tI zvY8sbk-WbR@&J6^Ma0QTDJ6=H^w9OudB_<`KvOPhCz^lIwEG zgZcn6D36e^;w!Zw!K+}L|EtKvD+Gh~-9At`)PXjfA0*z_ns86=1L%Af9DI2K0W1H6 zpi%7bAfR6jK2T)ARUgU4_5VmM14Pdwi{-k(5EHMR^f)|5S0SC%4uCp|IzXM}@$uxs z=SV6@OI4w1S|xC;Q;-$>$gz2V=1ETmYcZ(;Eo9G88|>L_0%O!0o^@^?6-voio$b)Y>v-_~E|O zdjTW)UpTWL%nBMSAEm-7H4JD0 z7m}kxhVsBwIF+>koUMmxR}g~R;u>D-S8-)eRdJdM==vC01`DFXSqU5-22XAf#9;s} z`(LQ57U=fn@qN8y(_@0j7CHmn!woeE(*tAcQ8=?i0F<}a_a8#C&fqR%F3}j)3qP&#CjygAPq@N6|aHFDqa9o$8JM|TqJl$t^|Xl zxL7Ar3+!)Jv`Lgn5WH5x&(5h0&E*&4709q$CO)8lO!o|sDjaK;^d z&q#JJe0K&lX~@GL(_pEmCG-s1d;6X$KrfQgUL>(4177UW*ut$HT$DF&=yjcGHd9+I zWjtkV>sc_9lvO%`EOV&y+1||O9n$ZR>F{kh%URXF;r!y(_ZMG$Kj6W{%LSq2 zY~Va0FKI%=MPY-^WL@~adSDK3VN^<_bsii>Z?sEuTZ0aj)Yolrz%qyVivWDQrQxX5 zMcjZ7Sg`t1ubkmVHm?am0<%EUwoW-S#@I%j1EE?KWpd}KELfv1TPDd671moS!5pZs z0D4`^Q$Z4Jzcm-oa_N`$qTXx6L5!1qk+mESucEUXyo$1ecK}9-1QltQ9+QSGbIKkV z13JLUuKnt?ormHCV)rrSQma?QcS{a@y~Gg?sRPa_zI8I6*hG_z15g?_OH3)O*Zrty zhZDLsY$!#n!)^`1jsX-q8V-9X4Uo@9L>e(1r0hdf(NQOP<@@RQT^Y)R&f09_d5CEUkI&x z2)FAKxLyD6503zN!H&Y+@_+x|vFraEOPUsEyZ--1S;Ck1|9_i+R^^&{^2T(?xzoM> Q%YVIyHPR{6a(wuI00Z?7g8%>k literal 0 HcmV?d00001 diff --git a/evidence/final-state.json b/evidence/final-state.json new file mode 100644 index 00000000..1177a602 --- /dev/null +++ b/evidence/final-state.json @@ -0,0 +1,35 @@ +{ + "attachments": [ + { + "state": "ready", + "count": 384 + } + ], + "jobs": [ + { + "state": "ready", + "pipeline_version": 1, + "count": 380 + }, + { + "state": "superseded", + "pipeline_version": 1, + "count": 4 + }, + { + "state": "ready", + "pipeline_version": 2, + "count": 7 + } + ], + "replays": 2, + "load_observation_errors": [ + { + "id": "63d3337b-5ac0-478b-9e30-cde11dd2fd07", + "state": "ready", + "published_pipeline_version": 1, + "target_pipeline_version": 1 + } + ], + "violations": 0 +} \ No newline at end of file diff --git a/evidence/isolation.txt b/evidence/isolation.txt new file mode 100644 index 00000000..2817a2d5 --- /dev/null +++ b/evidence/isolation.txt @@ -0,0 +1,2 @@ +PASS: negative_upload_acl_and_24_concurrent_messages +PASS: accepted_survives_api_broker_restart diff --git a/evidence/lab4-baseline.json b/evidence/lab4-baseline.json new file mode 100644 index 00000000..8eee394f --- /dev/null +++ b/evidence/lab4-baseline.json @@ -0,0 +1,126 @@ +{ + "head": "8f112e092a10d72a98c3d9d4554c52b81ebff2b3", + "branch": "solution/BVT2301-lyakhonov", + "status": "", + "files": { + ".dockerignore": "9c307884eda8387a781e8942b4df571ae0eee950c822aff044e7ed3d21e325f9", + ".env.example": "292e0f2d8d4db97ec3cd9826db8f8bd37fcf310bb27cbc341645e36de5046bdf", + ".gitattributes": "f3c03226a7748b6a723a20549727f38b6fdea151f177fd7c1901c9af7151640a", + ".github/pull_request_template.md": "416abfd7d727759a69eed876b415079816e6fcf458e261dcac78cc719709d66f", + ".github/workflows/application.yml": "0d45cb1d8fa2a45ef8a368892b833b26d300c9ccc7310bf7930c2a3fd9227164", + ".github/workflows/contracts.yml": "57acaf40811dddc83f27bc52bff3e22e60f817ef1b71d764678562ec56f5ca97", + ".gitignore": "fd2c5f54cd9dde6efd64b3977cd10d1400f83609939585778a2acce1b087581d", + "CONTRIBUTING.md": "08f247085938bd1efc4b8f7f48e3d520b1dc11ce569f4259491e8f01b9abd02c", + "COURSE.md": "1e3c65e345b7f89d08ed86a3b50061935ee8157907cb4043cfa40689781f54e0", + "Dockerfile": "c2743d8778c97cfd43d991d2a6b4a92e4560672af4c4eae86565cd1fa9c21f78", + "Dockerfile.example": "37050a40f15e29ee7f5eed7b2123fea9e6531d68b5735256e4c41c60255cffc9", + "Dockerfile.s3": "384837c64e421e67f81355cafa0af0b32157fe54451b886e471f69af8e0a0285", + "MAINTAINERS.md": "86153124dc51f97e29b9f7fd17d0a7100571bf584ae8167f162cd9728451c124", + "Makefile": "15525fe37647b398715597764a4ff6bfbcd7b46515e5a983ff013ec01dfe302a", + "README.md": "db1ff101968c21d6ada8972c1f4708936457af2fc1fa37ff7918137a83de3860", + "REPORT.md": "a5311b5fbbb1c21871ecc2a6ea16267453fc22c5c5121f03677e48524016885b", + "compose.example.yaml": "f7c1ae414a65cabb4ec334293d237bedbf8b1a694bfd7a72cfd28a2667222d7c", + "compose.yaml": "4f6ee1613db8937a39d73aa45e23a4c3d9931ae990d7e8cf503bac4bf266db6c", + "contracts/README.md": "4c04faede4a2ffa35183455b51da21764b7fa3997c0a4b3963614b82586eed28", + "contracts/examples.json": "2816da2e63058d55bdcbeceb92b231ea2509c86df8c5a0aa908be83bd0599469", + "contracts/infrastructure.json": "18a6b38deab39878da174f01e54addc4c7de556f9b3b7f37d96ec1ef0dd240b7", + "contracts/openapi.json": "61c7dbac7bd8d57423a2883b4b7b1277e8e35918d25c37e2cd02eaf6817936f0", + "evidence/README.md": "4ae3ab8da9fe3af5442f6b2e60c55d7d34de7bf0addffcc6a55f981baf0fe5cf", + "evidence/attachments.txt": "f5890fcb8a2ddc7b165633adc0a4c34100d9cc3d4261bb217af58e6aa2730a99", + "evidence/audit.txt": "d531211f514c58865dc2dfe19764cab7d54d41606fe9dc39e7e47de8aec8cb78", + "evidence/bootstrap-repeat.txt": "f7e10da821041ce9968ca4b0218c936f154ed02414443abaa4f8b1c0ab03d8c5", + "evidence/check.txt": "695ccac9f75e5d2118641ba2b3f386e916796271477d7bc5e98700d67371624d", + "evidence/clean-export.json": "34ef3e5865d6f8edd80ea14503117534da193eb43d5a5f8ff68abf28b86ac8ff", + "evidence/cpu.json": "aafd0ff20c15b4e30532823185578cdd86aa00050231e1ba72d868221a5abf44", + "evidence/docker-resources.json": "d44b532980090157607205f726b7799fd8a8da612a722a6d1a0b829618f8e1de", + "evidence/failover-1.txt": "e354ceccdbbf099a9adb922fdda0cc8576d7cbe9a3cb5a47dddd396a90559c16", + "evidence/failover-2.txt": "3cfe80f5864aa30972075db20fb2c0c02bc3dc3a74a9e4b975b816a4797b8c07", + "evidence/figures/reconcile-cost.png": "b84d7d37da4e9791941f1288f14f1155a687824a76cf3cf89a82410ed890dce9", + "evidence/figures/reconcile-cost.svg": "3c27e170f99738946899995808e5fdcb04e0ed27c090fd7c943a7592b94e20da", + "evidence/figures/upload-memory.png": "5c3a8e0e008741b67d949b1eb2b976bb684bb0fcde1bb8bd7a7e05c679c19885", + "evidence/figures/upload-memory.svg": "c45b7952a662ae6938f7299b9a9b876d08b3301aabfabe5ce496b85d7d70110c", + "evidence/figures/upload-throughput.png": "32bcd3c589d7e21a8b6d9cfe339806df8847e9af4ab105c551645110b1c0576f", + "evidence/figures/upload-throughput.svg": "9fc4c2946a74f047f5f0c83291ba77f05653e3344c34e9c99f0eea7d994deeed", + "evidence/final-check-test.txt": "cc1c058c861dc83a1202ed2b81675a4042a46a0c203d45f27ce5c99004aa18b3", + "evidence/hardware.json": "52bcfcbf973798f88ca4a441af5a7b0122e69a21355a802c92dcc2a5544ae912", + "evidence/level4.json": "3752d90c6e5d4e5cb7c5b8db6124873e9b87af642f3ea98a82240241937540d7", + "evidence/make-check-test.txt": "cc1c058c861dc83a1202ed2b81675a4042a46a0c203d45f27ce5c99004aa18b3", + "evidence/operations.json": "c222ef078951ad6e3a40515bd8a56188d0a0c2b1c1de8ba92f9f98976a235216", + "evidence/operations.txt": "b6bfd6e303838f66a333904a6fae16920a66e13cad90709e595859d79c0ab78a", + "evidence/persistence.txt": "e13f00023e004d83b2dfa575242fd3823985ac04e9e4107fb714acf0ccee07bd", + "evidence/reconcile.json": "a2150e0ed2951ba01e75c00c7257c8e285875aa58b3145354beab5a634f69b1a", + "evidence/repro-build.txt": "0300c2aa5019aaaa611960d82580bdfd8f53c540c494c00a2f40b73ce2b895bd", + "evidence/runtime.json": "d810634b8aed4c25c52ef9a0ad1a8fa7d73f76d1083b10a52ac260149d772616", + "evidence/s3-source-hashes.json": "9d0d1f414944cd6f0662067b9b2aa1c7fab7755baaa7b5560cd8740666e10f64", + "evidence/smoke.txt": "85fe65743939225bcec55093d4338aefd64679b5e457f5a6c8c07adff32d0743", + "evidence/sources-manifest.json": "44e11ad93dd156b2ebdfd99e91f2766400ffe4c5573701bebe37ee7b69b0a785", + "evidence/summary-output.txt": "2f7e52f1ac2a00d5999813149db612ba51caab3d043330974b598e1ed1528e09", + "evidence/summary.json": "4182e194ed6d12effd557bd5263b9f5a7fecfc5c6c8510b6214c183b4f808d2d", + "evidence/transfer.md": "3f874f17971915ba929aa3c31fdc79762d98baebfdf387cfcd327e78bb81d852", + "evidence/upload-load-output.txt": "bdeaae801cf295bb97b87ac3b8ef7878a7a0679d36bb43d5a1dc49d84b5da6d7", + "evidence/upload-load.json": "d965adbcfb4313e8529537129a65126ca17c8511758c91dd792b3b249b681e7d", + "evidence/upload-summary.csv": "9426de7d7c84be13bb85ef528fcea0da6cda9980e16445b19920f9129b768679", + "evidence/validate.txt": "69f17e4bbfdcc30ca307ad250746fef7007e94193586ea01e13e4040e1048e97", + "examples/requests.http": "53eea65721ef1c3e18470dbb95c894611c08856bab650fa0cfc0721f042cae91", + "infra/README.md": "33a4f7b870fd0b2bed98d46daf26d45dd7436176739672925246f8e3fe7c2278", + "infra/bucket.sh": "f7aed7c75255555c1b773adf2686d3206b113e5f0af863b96c833a63470370f6", + "infra/haproxy.cfg": "b359e64c9c1b923b85826db8c111a59fce4e26f788a784662b9a5046e65c226b", + "infra/reconciliation.md": "a3855637ffbdbfa06ef5f03744a068b9088a51e025f373330467f12a9aa27303", + "infra/s3-entrypoint.sh": "9b8220e117638e6a16c75cc6c824718209e7d5a767ed0b28f7038b58d676ec8c", + "infra/s3-policy.json": "463b042e97c4982be9b44ca22708c0d758d3f764c1cc7bb49710356a8549821b", + "lab.json": "30fd674b572fb0bf3a6845ce62987305db7f5b06d87f20aae72a1e30e566b185", + "lab4.env.example": "8c6da0b5707bb7e00c94fb3f626378412d335c0d6af2e29323e2e88323dc918c", + "migrations/001_initial.sql": "d4c044482ba61b6a59b35cb02cff16b650fa0fb813242c0678d45a3564e86340", + "migrations/002_idempotency.sql": "b3938061613446d1676dfdf41af1b8417dd0396d2a65e4d3d6b263d311ccdbce", + "migrations/003_attachments.sql": "4137877de095ca8cb590582bfa59758c31248f013e9b99ddcf6a2e2bedaff125", + "requirements-dev.txt": "e70f5e9db859ef3439dc6ae647890a18cd7a2752889598bbd19c822f31b7c499", + "requirements-research.txt": "1ff1ed1bc02424e20028427bd35d1b272c9700126033add35adf4566650dc067", + "requirements.lock.txt": "580f40ed0e8de26de95ac957e5b048a81d586c6d9861df5dfecd99700f57b072", + "requirements.txt": "dc14f80eca7ded138ec7fff542192d8013b74c458c3dd3234b11b6a391f75bf0", + "scripts/actions.py": "c58c1c5a72d474e38907440e0dea9914196bb0697cd1235c4b36e6335b151fdd", + "scripts/audit_solution.py": "609c740570364bec39f7d26bf85176537aea8e007fb21af5e0bedd12e6c60942", + "scripts/bootstrap.py": "db88a2aeba07281cd5ae8a001f0455bf8b5e19639bb06462513687fc59717aca", + "scripts/check_contract.py": "61e300fe5c1074360418e24b4653b12f96808247a2beaf2a64859aa360f81493", + "scripts/plot_results.py": "fecbeb33d73e7d3cbdeda41701f3be40dd6d04f974df1e85bd5c9a2152182c64", + "scripts/restart.cmd": "e008c8fa1ce5cacf582e502179ec78e15d45fe90dda1f4c929b7a01052fe3b11", + "scripts/restart.example.sh": "0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b", + "scripts/restart.sh": "fcea33ea66beeb9b2ca6964a00c817a45afb7626eb9e4deb92d03adfc78a70b2", + "scripts/run_upload_load.py": "d1959fe3756fecbaa1923c972dcd41678634100b5f59a10bcab97851cb9d25cf", + "scripts/stop-one.cmd": "95e787774e6ffb007dc53c94a24c91d4a6ff3c2ae2bb15198cf7d4ccb06664ea", + "scripts/stop-one.example.sh": "1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811", + "scripts/stop-one.sh": "82a0c50fd834c9ce330ec4b6b808c64c7d76dda7352ae4210a3322c9e0f2e125", + "scripts/validate_openapi.py": "cd7a33221b1860c5d2330e52afa6cb222d375db671ae9ab379ba51a0d3a77946", + "scripts/verify_clean_export.py": "a28a9388c416e101792b7587a784d127b2e7c78dd56da0bf9fa40d384533c52c", + "scripts/wait_ready.py": "29e9965053ad2cba45dc68a4cc462a7ccd2c6cbd9da9ed89351721f926f5643c", + "src/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "src/app.py": "0cb5d0151e73166aed98acfe151415a93f18221cf27ebd8e39f433cda30b8f13", + "src/cache.py": "bfc8f1aa90bee4625b3887a3487be085560b7a2705cc94dcb557dd6c03062028", + "src/config.py": "aafa2db508658d2d5467d2c57ed5f4dea11609ab5928f86abc1e35fc8ba923d7", + "src/main.py": "3f49c442758f1a4c3a156a4555f2b3f6d8dff0529fc62cb763f11eaaa2d2760b", + "src/migrate.py": "48c2819b34e22ee099a97d8742a65c87bc0abf9cd617cf6b6caae8a1ad145af5", + "src/models.py": "afc5b651059a440904e1c4ac509ac8e24796bceac5869a6421e42d388f2fe6d6", + "src/objects.py": "75f15c13be0b5eecf441a437a8250a0d6367b7b098b288a2cd1994c0b1a4321a", + "src/reconcile.py": "228a7994ce5b4110d7391734b0885c01ece754912c285c7f530427ec2674f133", + "src/routes/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "src/routes/attachments.py": "5518b8a264b8b0454e30606123119e83382159f2f031864e33a3e48a62496c84", + "src/routes/chats.py": "46ef834fce05037c2ecab26192ed58c28be81afc6a1db1375560f11bb9d63c12", + "src/routes/common.py": "fa2df76c819abe9a99b8b380a50b2aa4556aa43ac84afa95cac574aab8468a7e", + "src/routes/health.py": "a2d7aef51d4fff4f4c8523a4b1d1b894747f945ac3dd4d26b521d6a228cb88be", + "src/routes/messages.py": "7dc016d1252f07538c06b7c155d3efb8af02ac2a2a9867267168a74e7cf0bd3c", + "src/routes/users.py": "f5d4c7e4c4e5b1abfde7869a46e8a81a935b32f4809ddb8b2a4cb13175a24f50", + "src/storage.py": "81f5bcb0bc6ce356c76f1ea0b90d74566d436ca6f2dcc35e80cc0ae512588979", + "student-tests/README.md": "eb43baa6868c409f71905c7ab11b0bb187a8e9a87fc4842909949b270107e354", + "student-tests/test_attachments.py": "e26fece8adc6a63bcccfe6c1de53e3899f0c43d0c8c9fd6502c988403a9a95e5", + "student-tests/test_operations.py": "7a62d819afdaea80767cb2ff4e8141eebce9e564ec20241c7d3f5b645d742999", + "student-tests/test_reconcile.py": "f1e0f7461f57fa0aad2a16e118fe86ed5852ec1d86caac07ea81a829c7f2f787", + "student-tests/upload_load.py": "022eea2772fcbf780042b3959c516788929dfde4968dcd9f7caa2531999c52e9", + "tests/README.md": "89afb3063ec0ccd7cb68fdac1de57513f9cc81b77bad96f202daaca2b66a5e82", + "tests/client.py": "b318084e2a4c380db91a55d082190bf33979fa556a4c4ca87e01f46ae275775c", + "tests/fixtures/README.md": "0eaa4e8837032d1786fc0c4b1576656ff3f4d1931fdd51b80e5c053f88ab3872", + "tests/fixtures/sample.png": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "tests/image_probe.py": "ef48d8882ab138b9699a173e7419e5814c22a10b01b6b0ddbb50ea49481a5baa", + "tests/load.py": "9a9aff05e109630e243c3b092d7bd564344e4872b29e38b6618647a08460a72e", + "tests/resilience.py": "4ba30b9c81a0664127ddab2b0bea5e76aa2087be8bb25aa13cd5a8e69be23fdd", + "tests/smoke.py": "354e168e1e644145c3ff50bbbe189fd217cce5659aa125f588098025ffab0fdf" + } +} \ No newline at end of file diff --git a/evidence/load-messages-resources.json b/evidence/load-messages-resources.json new file mode 100644 index 00000000..8d62c6b8 --- /dev/null +++ b/evidence/load-messages-resources.json @@ -0,0 +1,775 @@ +{ + "command": [ + "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-X", + "utf8", + "tests/load.py", + "--base-url", + "http://127.0.0.1:8085", + "--duration", + "30", + "--concurrency", + "4", + "--output", + "evidence/load-messages.json" + ], + "exit_code": 0, + "samples": [ + { + "at_seconds": 3.241, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "34.76%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.93%", + "MemUsage": "108.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "182MB / 182MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "48.21%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "19.08%", + "MemUsage": "122.1MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "137MB / 137MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "1.63%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.71MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "288kB / 253kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "33.80%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "10.18%", + "MemUsage": "52.11MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "6.3MB / 8.09MB", + "PIDs": "18" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.77%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.56MB / 1.1MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.37%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.75%", + "MemUsage": "13.76MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "317MB / 317MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.62%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "0.84%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "479kB / 521kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.03%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.03%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 8.631, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "28.24%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.97%", + "MemUsage": "108.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "182MB / 183MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "50.33%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "19.13%", + "MemUsage": "122.5MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "137MB / 137MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "53.78%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.38%", + "MemUsage": "78.73MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "288kB / 253kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "24.36%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "10.65%", + "MemUsage": "54.51MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "7.27MB / 8.97MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.88%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.58MB / 1.11MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.03%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "11.15%", + "MemUsage": "14.27MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "318MB / 318MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.65%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "0.74%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "479kB / 521kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.04%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.05%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 13.955, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "46.72%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.99%", + "MemUsage": "108.8MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "183MB / 183MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "48.40%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "19.24%", + "MemUsage": "123.1MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "138MB / 138MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "42.48%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "24.05%", + "MemUsage": "123.1MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "288kB / 253kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "30.00%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "11.38%", + "MemUsage": "58.29MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "8.2MB / 9.81MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.88%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.61MB / 1.13MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.29%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "11.87%", + "MemUsage": "15.2MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "318MB / 319MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.46%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "0.66%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "479kB / 521kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.03%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.04%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 19.25, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "39.37%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "17.00%", + "MemUsage": "108.8MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "184MB / 184MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "49.09%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "19.17%", + "MemUsage": "122.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "138MB / 139MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "0.23%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.71MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "288kB / 253kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 1.57MB", + "CPUPerc": "21.35%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "11.24%", + "MemUsage": "57.52MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "9.04MB / 10.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.67%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.63MB / 1.14MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "16.23%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "11.01%", + "MemUsage": "14.09MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "319MB / 319MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.49%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "0.79%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385.1MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "479kB / 521kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.04%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.05%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 24.605, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "49.61%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "17.00%", + "MemUsage": "108.8MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "184MB / 185MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "31.04%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "19.17%", + "MemUsage": "122.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "139MB / 140MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "75.54%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.73%", + "MemUsage": "162.4MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "288kB / 253kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 1.57MB", + "CPUPerc": "32.34%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "11.36%", + "MemUsage": "58.14MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "10.5MB / 11.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.94%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.65MB / 1.16MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.43%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "11.71%", + "MemUsage": "14.98MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "320MB / 320MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.66%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "0.64%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "479kB / 521kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.03%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.04%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 29.874, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "36.39%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "17.01%", + "MemUsage": "108.8MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "185MB / 186MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "48.11%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "19.17%", + "MemUsage": "122.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "140MB / 140MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "4.78%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "19.12%", + "MemUsage": "97.92MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "288kB / 254kB", + "PIDs": "60" + }, + { + "BlockIO": "0B / 20.3MB", + "CPUPerc": "27.03%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "11.51%", + "MemUsage": "58.93MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "11.6MB / 12.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.47%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.68MB / 1.17MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.59%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "11.22%", + "MemUsage": "14.36MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "321MB / 321MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.47%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "0.59%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.98%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "480kB / 522kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.03%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.05%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/load-messages-run.txt b/evidence/load-messages-run.txt new file mode 100644 index 00000000..2e0439d9 --- /dev/null +++ b/evidence/load-messages-run.txt @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.112, + "concurrency": 4, + "successful_requests": 2507, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 83.256, + "successful_latency_ms": { + "p50": 25.341, + "p95": 146.956, + "p99": 251.292 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-messages.json b/evidence/load-messages.json new file mode 100644 index 00000000..2e0439d9 --- /dev/null +++ b/evidence/load-messages.json @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.112, + "concurrency": 4, + "successful_requests": 2507, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 83.256, + "successful_latency_ms": { + "p50": 25.341, + "p95": 146.956, + "p99": 251.292 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-pipeline.json b/evidence/load-pipeline.json new file mode 100644 index 00000000..ea87a1ae --- /dev/null +++ b/evidence/load-pipeline.json @@ -0,0 +1,3520 @@ +[ + { + "profile": "320x200", + "bytes_per_image": 111817, + "repeat": 1, + "concurrency": 2, + "requested_duration": 20, + "elapsed_seconds": 20.093, + "warmup_completed": 1, + "completed": 80, + "successful_rps": 3.982, + "failures": [], + "error_fraction": 0.0, + "p50": 0.401, + "p95": 0.7265, + "p99": 0.7436, + "upload_p95": 0.0781, + "stats": [ + { + "at_seconds": 3.186, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "6.10%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.58%", + "MemUsage": "99.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "1.31MB / 1.41MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "16.94%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.44%", + "MemUsage": "98.81MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "505kB / 509kB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 106kB", + "CPUPerc": "0.57%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.35%", + "MemUsage": "78.61MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "22.4kB / 15.1kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "6.94%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.24%", + "MemUsage": "37.06MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "2.23MB / 2.32MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.05%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "480kB / 304kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.12%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.15%", + "MemUsage": "12.99MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "3.03MB / 3MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.64%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 2.79MB", + "CPUPerc": "5.22%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "44.74%", + "MemUsage": "343.6MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "2.98MB / 1.69MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.51%", + "MemUsage": "57.63MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "13kB / 9.44kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.33%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.05%", + "MemUsage": "61.84MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "621kB / 676kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.17%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "7.82%", + "MemUsage": "60.04MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "621kB / 676kB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 8.271, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.26%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.75%", + "MemUsage": "100.8MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "2.44MB / 2.54MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.36%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.72%", + "MemUsage": "100.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "2MB / 1.89MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 106kB", + "CPUPerc": "75.91%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.69%", + "MemUsage": "162.3MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "38.2kB / 29kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "5.28%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.43%", + "MemUsage": "38.06MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "2.44MB / 2.62MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.10%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "533kB / 344kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.37%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.30%", + "MemUsage": "13.18MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "5.29MB / 5.26MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.36%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 7.52MB", + "CPUPerc": "12.12%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "45.26%", + "MemUsage": "347.6MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "7.68MB / 4.1MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.96%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.52%", + "MemUsage": "57.64MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "50.7kB / 50.9kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "23.37%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.17%", + "MemUsage": "62.73MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "1.84MB / 2.01MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "28.57%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "7.93%", + "MemUsage": "60.92MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "1.84MB / 2.01MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 13.371, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "6.80%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.87%", + "MemUsage": "101.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "3.57MB / 3.67MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.67%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.71%", + "MemUsage": "100.5MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "3.36MB / 3.38MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 106kB", + "CPUPerc": "1.38%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.36%", + "MemUsage": "78.62MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "54.3kB / 43.5kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "7.67%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.79%", + "MemUsage": "39.87MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "2.64MB / 2.91MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.24%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "586kB / 385kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.73%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.34%", + "MemUsage": "13.23MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "7.67MB / 7.64MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.47%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 12.6MB", + "CPUPerc": "8.82%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "45.38%", + "MemUsage": "348.5MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "12.7MB / 6.61MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.52%", + "MemUsage": "57.64MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "50.7kB / 50.9kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "27.77%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.20%", + "MemUsage": "63.01MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "3.18MB / 3.35MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "26.44%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "7.96%", + "MemUsage": "61.1MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "3.07MB / 3.34MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 18.42, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "5.51%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.88%", + "MemUsage": "101.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "4.59MB / 4.69MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "5.27%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.87%", + "MemUsage": "101.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "4.95MB / 4.98MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "1.08%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.36%", + "MemUsage": "78.62MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "70.9kB / 58kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "3.68%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.87%", + "MemUsage": "40.29MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "2.83MB / 3.21MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.23%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "640kB / 426kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.97%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.84%", + "MemUsage": "13.87MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "10.3MB / 10.2MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.44%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 17.8MB", + "CPUPerc": "7.51%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "41.96%", + "MemUsage": "322.3MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "18MB / 9.13MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.52%", + "MemUsage": "57.64MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "50.7kB / 50.9kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "30.74%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.31%", + "MemUsage": "63.84MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "4.41MB / 4.81MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "31.68%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.08%", + "MemUsage": "62.02MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "4.41MB / 4.81MB", + "PIDs": "5" + } + ] + } + ] + }, + { + "profile": "320x200", + "bytes_per_image": 111817, + "repeat": 2, + "concurrency": 2, + "requested_duration": 20, + "elapsed_seconds": 20.24, + "warmup_completed": 1, + "completed": 73, + "successful_rps": 3.607, + "failures": [], + "error_fraction": 0.0, + "p50": 0.4219, + "p95": 0.7366, + "p99": 0.7857, + "upload_p95": 0.0893, + "stats": [ + { + "at_seconds": 2.572, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "18.46%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.88%", + "MemUsage": "101.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "5.94MB / 6.04MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.80%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.98%", + "MemUsage": "102.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "5.96MB / 5.99MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "75.65%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.96%", + "MemUsage": "163.7MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "84.9kB / 70.6kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "4.89%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.98%", + "MemUsage": "40.88MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3MB / 3.47MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.67%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.70%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "688kB / 462kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.84%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.37%", + "MemUsage": "13.27MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "12.4MB / 12.4MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.50%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.80%", + "MemUsage": "3.59MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 22.4MB", + "CPUPerc": "8.34%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "42.04%", + "MemUsage": "322.9MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "22.5MB / 11.3MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.52%", + "MemUsage": "57.64MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "50.7kB / 50.9kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "16.56%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.42%", + "MemUsage": "64.67MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "5.52MB / 6.01MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "32.08%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.18%", + "MemUsage": "62.84MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "5.52MB / 6.01MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 7.579, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.71%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.90%", + "MemUsage": "101.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "7.17MB / 7.28MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.33%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.98%", + "MemUsage": "102.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "6.74MB / 6.76MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "1.11%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.36%", + "MemUsage": "78.63MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "97.9kB / 82.3kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "7.51%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.94%", + "MemUsage": "40.66MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3.16MB / 3.71MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.89%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "734kB / 497kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.68%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.57%", + "MemUsage": "13.53MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "14.3MB / 14.3MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.47%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 26.3MB", + "CPUPerc": "5.38%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "42.10%", + "MemUsage": "323.3MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "26.4MB / 13.3MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.52%", + "MemUsage": "57.64MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "50.7kB / 50.9kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "21.67%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "9.74%", + "MemUsage": "74.78MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "6.61MB / 7.08MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "21.32%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.24%", + "MemUsage": "63.3MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "6.49MB / 6.95MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 12.595, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.93%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.90%", + "MemUsage": "101.8MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "8.19MB / 8.29MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.27%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.99%", + "MemUsage": "102.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "8.09MB / 8.11MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "0.91%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.36%", + "MemUsage": "78.64MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "113kB / 95.5kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "3.37%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.91%", + "MemUsage": "40.48MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3.34MB / 3.98MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.35%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "784kB / 535kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.41%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.40%", + "MemUsage": "13.31MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "16.6MB / 16.6MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.47%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 30.9MB", + "CPUPerc": "6.08%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "42.27%", + "MemUsage": "324.6MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "31MB / 15.6MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.52%", + "MemUsage": "57.64MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "50.7kB / 50.9kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 262kB", + "CPUPerc": "19.44%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.50%", + "MemUsage": "65.27MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "7.71MB / 8.28MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "32.45%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.24%", + "MemUsage": "63.27MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "7.71MB / 8.28MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 17.542, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "8.61%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.99%", + "MemUsage": "102.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "9.31MB / 9.41MB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "8.94%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.00%", + "MemUsage": "102.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "9.33MB / 9.35MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "75.17%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.62%", + "MemUsage": "161.9MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "127kB / 108kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "15.41%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.92%", + "MemUsage": "40.55MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3.52MB / 4.26MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.32%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "831kB / 571kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.60%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.42%", + "MemUsage": "13.33MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "18.9MB / 18.8MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.50%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 35.7MB", + "CPUPerc": "16.20%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "38.31%", + "MemUsage": "294.2MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "35.8MB / 17.8MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.80%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.62%", + "MemUsage": "57.9MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "105kB / 85.3kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 262kB", + "CPUPerc": "26.49%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.45%", + "MemUsage": "64.89MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "8.82MB / 9.61MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "28.54%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.21%", + "MemUsage": "63.05MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "8.82MB / 9.61MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 22.662, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.38%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.03%", + "MemUsage": "102.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "9.95MB / 10.1MB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.20%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.00%", + "MemUsage": "102.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "10.1MB / 10.1MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "0.27%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.41%", + "MemUsage": "78.89MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "136kB / 116kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "0.37%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.97%", + "MemUsage": "40.79MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3.7MB / 4.49MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.43%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "872kB / 601kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.25%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.39%", + "MemUsage": "13.29MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "20.2MB / 20.2MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.53%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 38.5MB", + "CPUPerc": "0.53%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "39.63%", + "MemUsage": "304.3MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "38.7MB / 19.3MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.63%", + "MemUsage": "57.95MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "205kB / 220kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 262kB", + "CPUPerc": "0.04%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.45%", + "MemUsage": "64.9MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "9.55MB / 10.4MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.04%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.21%", + "MemUsage": "63.07MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "9.55MB / 10.4MB", + "PIDs": "5" + } + ] + } + ] + }, + { + "profile": "320x200", + "bytes_per_image": 111817, + "repeat": 3, + "concurrency": 2, + "requested_duration": 20, + "elapsed_seconds": 20.004, + "warmup_completed": 1, + "completed": 71, + "successful_rps": 3.549, + "failures": [], + "error_fraction": 0.0, + "p50": 0.4389, + "p95": 0.7694, + "p99": 0.82, + "upload_p95": 0.089, + "stats": [ + { + "at_seconds": 3.167, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.89%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.02%", + "MemUsage": "102.5MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "10.6MB / 10.7MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.55%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.01%", + "MemUsage": "102.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "10.6MB / 10.6MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "1.08%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.68MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "143kB / 123kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "4.92%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.97%", + "MemUsage": "40.79MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3.79MB / 4.63MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.90%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "902kB / 623kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.74%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.83%", + "MemUsage": "13.86MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "21MB / 21MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.44%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.80%", + "MemUsage": "3.59MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 40.6MB", + "CPUPerc": "4.68%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "39.66%", + "MemUsage": "304.6MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "40.8MB / 20.4MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.63%", + "MemUsage": "57.95MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "205kB / 220kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 262kB", + "CPUPerc": "18.74%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.50%", + "MemUsage": "65.31MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "10.2MB / 10.9MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "20.28%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.21%", + "MemUsage": "63.08MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "10MB / 10.9MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 8.313, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.47%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.02%", + "MemUsage": "102.5MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "11.7MB / 11.8MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.72%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.83%", + "MemUsage": "107.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "11.8MB / 11.9MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "72.76%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.15%", + "MemUsage": "159.5MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "157kB / 135kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "3.24%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.98%", + "MemUsage": "40.88MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3.97MB / 4.9MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.37%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "944kB / 655kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.82%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.48%", + "MemUsage": "13.41MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "23.5MB / 23.5MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.46%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 45.2MB", + "CPUPerc": "12.60%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "39.75%", + "MemUsage": "305.3MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "45.5MB / 22.5MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.63%", + "MemUsage": "57.95MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "205kB / 220kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 262kB", + "CPUPerc": "19.14%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.46%", + "MemUsage": "64.94MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "11.3MB / 12.3MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.39%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.22%", + "MemUsage": "63.09MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "11.1MB / 12.1MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 13.541, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "22.94%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.04%", + "MemUsage": "102.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "12.7MB / 12.8MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.37%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.00%", + "MemUsage": "102.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "12.6MB / 12.6MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "1.14%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.68MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "172kB / 149kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "7.65%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.07%", + "MemUsage": "41.32MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.14MB / 5.16MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.14%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1MB / 698kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.15%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.43%", + "MemUsage": "13.36MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "25.7MB / 25.6MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.44%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.80%", + "MemUsage": "3.59MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 49.7MB", + "CPUPerc": "7.57%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "39.82%", + "MemUsage": "305.8MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "49.9MB / 24.8MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.63%", + "MemUsage": "57.95MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "205kB / 220kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 262kB", + "CPUPerc": "33.11%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.46%", + "MemUsage": "64.96MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "12.4MB / 13.5MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "23.90%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "9.16%", + "MemUsage": "70.37MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "12.4MB / 13.3MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 18.863, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.62%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.03%", + "MemUsage": "102.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "13.9MB / 14.1MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "6.32%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.00%", + "MemUsage": "102.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "14.2MB / 14.2MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 143kB", + "CPUPerc": "0.92%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.68MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "189kB / 164kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "4.02%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.02%", + "MemUsage": "41.04MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.34MB / 5.46MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.10%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.06MB / 741kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.78%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.47%", + "MemUsage": "13.4MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "28.3MB / 28.3MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.48%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.80%", + "MemUsage": "3.59MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 54.9MB", + "CPUPerc": "7.08%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "39.94%", + "MemUsage": "306.8MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "55.1MB / 27.3MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.63%", + "MemUsage": "57.95MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "205kB / 220kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 389kB", + "CPUPerc": "24.86%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.80%", + "MemUsage": "67.59MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "13.7MB / 14.8MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "33.49%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.22%", + "MemUsage": "63.16MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "13.3MB / 14.5MB", + "PIDs": "5" + } + ] + } + ] + }, + { + "profile": "1024x768", + "bytes_per_image": 2360847, + "repeat": 1, + "concurrency": 2, + "requested_duration": 20, + "elapsed_seconds": 20.567, + "warmup_completed": 1, + "completed": 45, + "successful_rps": 2.188, + "failures": [], + "error_fraction": 0.0, + "p50": 0.8308, + "p95": 1.1487, + "p99": 1.1615, + "upload_p95": 0.2497, + "stats": [ + { + "at_seconds": 3.238, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "17.26%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.74%", + "MemUsage": "107.1MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "21.7MB / 21.8MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.95%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.71%", + "MemUsage": "106.9MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "24MB / 24.1MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 143kB", + "CPUPerc": "73.57%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.16%", + "MemUsage": "159.5MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "197kB / 172kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "6.43%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.05%", + "MemUsage": "41.2MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.46MB / 5.65MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.06%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.1MB / 769kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.77%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.48%", + "MemUsage": "13.41MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "45.4MB / 45.4MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.54%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.80%", + "MemUsage": "3.59MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 87.6MB", + "CPUPerc": "15.62%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "40.12%", + "MemUsage": "308.1MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "87.8MB / 44.5MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.63%", + "MemUsage": "57.95MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "205kB / 220kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 389kB", + "CPUPerc": "31.88%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "10.69%", + "MemUsage": "82.09MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "23.5MB / 24.1MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "31.23%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.27%", + "MemUsage": "63.51MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "18.7MB / 18.2MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 8.467, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.21%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.76%", + "MemUsage": "107.2MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "38.4MB / 38.5MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.69%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.72%", + "MemUsage": "107MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "36MB / 36MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 143kB", + "CPUPerc": "0.61%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.68MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "207kB / 180kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "2.56%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.07%", + "MemUsage": "41.29MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.65MB / 5.9MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.85%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.14MB / 798kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.74%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.69%", + "MemUsage": "13.68MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "74MB / 74MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.38%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 152MB", + "CPUPerc": "15.77%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "42.55%", + "MemUsage": "326.8MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "152MB / 73.1MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "329kB / 355kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 389kB", + "CPUPerc": "38.39%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "11.89%", + "MemUsage": "91.32MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "37.7MB / 41.9MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "23.82%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "11.46%", + "MemUsage": "87.98MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "35.4MB / 38.9MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 13.441, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "19.80%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "17.20%", + "MemUsage": "110.1MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "62.2MB / 62.3MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "6.72%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.72%", + "MemUsage": "107MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "38.4MB / 38.5MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "20.44%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.68MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "215kB / 187kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "6.24%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.17%", + "MemUsage": "41.83MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.77MB / 6.08MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.71%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.17MB / 825kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.83%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.50%", + "MemUsage": "13.44MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "100MB / 100MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.57%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 210MB", + "CPUPerc": "13.70%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "42.87%", + "MemUsage": "329.2MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "211MB / 96.9MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "329kB / 355kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 389kB", + "CPUPerc": "33.90%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "13.01%", + "MemUsage": "99.91MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "49.6MB / 56.7MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 2.52MB", + "CPUPerc": "27.32%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.25%", + "MemUsage": "63.33MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "47.3MB / 56.8MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 18.47, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "19.08%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.76%", + "MemUsage": "107.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "76.6MB / 76.7MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.38%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.72%", + "MemUsage": "107MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "52.7MB / 52.8MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "56.27%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "29.65%", + "MemUsage": "151.8MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "224kB / 196kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "3.83%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.13%", + "MemUsage": "41.61MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.91MB / 6.27MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.42%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.70%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.21MB / 854kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.01%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.50%", + "MemUsage": "13.44MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "129MB / 129MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.50%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 274MB", + "CPUPerc": "10.91%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "43.17%", + "MemUsage": "331.6MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "274MB / 125MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "329kB / 355kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 389kB", + "CPUPerc": "50.93%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "11.50%", + "MemUsage": "88.29MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "63.9MB / 74.4MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 2.52MB", + "CPUPerc": "37.36%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.26%", + "MemUsage": "63.46MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "61.5MB / 74.6MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 23.445, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.57%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.76%", + "MemUsage": "107.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "81.4MB / 81.5MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.59%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.72%", + "MemUsage": "107MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "57.5MB / 57.6MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "0.24%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.69MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "228kB / 199kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "2.43%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.15%", + "MemUsage": "41.73MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.98MB / 6.38MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.36%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.24MB / 873kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.28%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.47%", + "MemUsage": "13.41MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "138MB / 138MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.55%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 301MB", + "CPUPerc": "0.50%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "43.86%", + "MemUsage": "336.9MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "302MB / 137MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "329kB / 355kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 389kB", + "CPUPerc": "0.03%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.53%", + "MemUsage": "65.5MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "68.6MB / 83.3MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 2.52MB", + "CPUPerc": "0.04%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.28%", + "MemUsage": "63.62MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "68.6MB / 83.5MB", + "PIDs": "5" + } + ] + } + ] + }, + { + "profile": "1024x768", + "bytes_per_image": 2360847, + "repeat": 2, + "concurrency": 2, + "requested_duration": 20, + "elapsed_seconds": 20.396, + "warmup_completed": 1, + "completed": 32, + "successful_rps": 1.569, + "failures": [ + "Failure: GET /api/v1/attachments/63d3337b-5ac0-478b-9e30-cde11dd2fd07: connection/TLS failure (URLError)" + ], + "error_fraction": 0.030303030303030304, + "p50": 0.79, + "p95": 1.1397, + "p99": 1.2381, + "upload_p95": 0.1837, + "stats": [ + { + "at_seconds": 3.161, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.23%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.76%", + "MemUsage": "107.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "88.5MB / 88.6MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "5.61%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.72%", + "MemUsage": "107MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "62.3MB / 62.4MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "75.36%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.14%", + "MemUsage": "159.5MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "232kB / 203kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "1.58%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.08%", + "MemUsage": "41.39MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.03MB / 6.46MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.77%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.27MB / 890kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.48%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.50%", + "MemUsage": "13.44MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "150MB / 150MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.51%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 325MB", + "CPUPerc": "5.04%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "44.82%", + "MemUsage": "344.2MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "325MB / 149MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "329kB / 355kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 2.75MB", + "CPUPerc": "13.48%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "10.71%", + "MemUsage": "82.23MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "75.7MB / 89.2MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 4.91MB", + "CPUPerc": "32.77%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.30%", + "MemUsage": "63.75MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "73.4MB / 89.4MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 8.318, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.20%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.78%", + "MemUsage": "107.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "98.1MB / 98.2MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.72%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "18.13%", + "MemUsage": "116MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "64.7MB / 64.8MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "1.08%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.69MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "237kB / 207kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "5.04%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.14%", + "MemUsage": "41.66MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.12MB / 6.58MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.54%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.3MB / 911kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.32%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.49%", + "MemUsage": "13.43MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "165MB / 165MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.37%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 360MB", + "CPUPerc": "6.87%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "44.98%", + "MemUsage": "345.4MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "360MB / 163MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "329kB / 355kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 2.75MB", + "CPUPerc": "23.99%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.55%", + "MemUsage": "65.7MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "82.9MB / 101MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 4.91MB", + "CPUPerc": "18.08%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.31%", + "MemUsage": "63.82MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "80.5MB / 98.3MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 13.639, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.42%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.83%", + "MemUsage": "107.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "115MB / 115MB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "18.62%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.72%", + "MemUsage": "107MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "71.9MB / 72MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "0.55%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.7MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "243kB / 213kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "9.98%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.14%", + "MemUsage": "41.68MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.22MB / 6.72MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.83%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.33MB / 937kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.29%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.53%", + "MemUsage": "13.48MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "186MB / 186MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.48%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 402MB", + "CPUPerc": "11.15%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "46.76%", + "MemUsage": "359.1MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "402MB / 185MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.36%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "400kB / 435kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 2.75MB", + "CPUPerc": "46.72%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "13.07%", + "MemUsage": "100.4MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "92.4MB / 110MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 4.91MB", + "CPUPerc": "29.55%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "9.31%", + "MemUsage": "71.53MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "90MB / 107MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 18.881, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.00%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "17.06%", + "MemUsage": "109.2MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "127MB / 127MB", + "PIDs": "18" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "24.09%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "17.08%", + "MemUsage": "109.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "83.8MB / 81.6MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "76.02%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.11%", + "MemUsage": "159.3MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "252kB / 221kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 811kB", + "CPUPerc": "4.71%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.15%", + "MemUsage": "41.73MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.38MB / 6.95MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.08%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.37MB / 964kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.26%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.53%", + "MemUsage": "13.47MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "210MB / 210MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.51%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 461MB", + "CPUPerc": "19.87%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "49.59%", + "MemUsage": "380.9MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "462MB / 209MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "400kB / 435kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "39.48%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.56%", + "MemUsage": "65.73MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "104MB / 128MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 7.87MB", + "CPUPerc": "37.49%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.88MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "104MB / 128MB", + "PIDs": "5" + } + ] + } + ] + }, + { + "profile": "1024x768", + "bytes_per_image": 2360847, + "repeat": 3, + "concurrency": 2, + "requested_duration": 20, + "elapsed_seconds": 20.969, + "warmup_completed": 1, + "completed": 40, + "successful_rps": 1.908, + "failures": [], + "error_fraction": 0.0, + "p50": 1.0675, + "p95": 1.2299, + "p99": 1.3224, + "upload_p95": 0.2471, + "stats": [ + { + "at_seconds": 3.248, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "24.77%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.82%", + "MemUsage": "107.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "143MB / 144MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.43%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "17.64%", + "MemUsage": "112.9MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "93.4MB / 93.6MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "0.85%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.7MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "260kB / 228kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 811kB", + "CPUPerc": "6.77%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.14%", + "MemUsage": "41.68MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.5MB / 7.13MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.14%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.41MB / 994kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.60%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.56%", + "MemUsage": "13.51MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "236MB / 236MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.56%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 514MB", + "CPUPerc": "11.92%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "49.65%", + "MemUsage": "381.3MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "514MB / 235MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "400kB / 435kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "39.45%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "9.07%", + "MemUsage": "69.68MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "119MB / 142MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 7.87MB", + "CPUPerc": "53.17%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "11.88%", + "MemUsage": "91.25MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "116MB / 140MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 8.179, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.43%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.82%", + "MemUsage": "107.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "153MB / 153MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "19.29%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.78%", + "MemUsage": "107.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "108MB / 108MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "1.60%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.7MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "268kB / 235kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "6.97%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.15%", + "MemUsage": "41.7MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.62MB / 7.3MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.42%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.44MB / 1.02MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.22%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.96%", + "MemUsage": "14.03MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "260MB / 260MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.50%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 567MB", + "CPUPerc": "11.16%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "49.76%", + "MemUsage": "382.2MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "568MB / 256MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "400kB / 435kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "35.85%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.56%", + "MemUsage": "65.78MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "128MB / 157MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 7.87MB", + "CPUPerc": "46.26%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "10.35%", + "MemUsage": "79.46MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "128MB / 154MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 13.271, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.22%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.82%", + "MemUsage": "107.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "167MB / 168MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "22.82%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.75%", + "MemUsage": "107.2MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "115MB / 115MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "77.58%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "30.99%", + "MemUsage": "158.7MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "275kB / 241kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "3.48%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.15%", + "MemUsage": "41.73MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.74MB / 7.48MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.92%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.48MB / 1.04MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.66%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.57%", + "MemUsage": "13.53MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "281MB / 281MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.57%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 618MB", + "CPUPerc": "11.88%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "49.86%", + "MemUsage": "382.9MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "618MB / 278MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "400kB / 435kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "44.75%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.79MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "140MB / 172MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "45.42%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.91MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "138MB / 169MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 18.168, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.55%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.81%", + "MemUsage": "107.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "177MB / 177MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.46%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.75%", + "MemUsage": "107.2MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "127MB / 127MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "0.57%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.71MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "282kB / 248kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "6.22%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.19%", + "MemUsage": "41.95MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.85MB / 7.64MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.82%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.52MB / 1.07MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.72%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "11.17%", + "MemUsage": "14.3MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "303MB / 303MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.45%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 663MB", + "CPUPerc": "10.01%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.06%", + "MemUsage": "384.4MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "663MB / 301MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "400kB / 435kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "36.46%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "13.11%", + "MemUsage": "100.7MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "152MB / 184MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "35.49%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "12.87%", + "MemUsage": "98.85MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "149MB / 181MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 23.234, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.76%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.82%", + "MemUsage": "107.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "182MB / 182MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.56%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.75%", + "MemUsage": "107.2MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "136MB / 137MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "65.46%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.71MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "287kB / 253kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "0.94%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.15%", + "MemUsage": "41.71MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.97MB / 7.81MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.52%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.70%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.55MB / 1.09MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.26%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.56%", + "MemUsage": "13.51MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "317MB / 317MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.45%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "1.58%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.82%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "479kB / 521kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.04%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.04%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + } + ] + } +] \ No newline at end of file diff --git a/evidence/load-run.txt b/evidence/load-run.txt new file mode 100644 index 00000000..abbcc961 --- /dev/null +++ b/evidence/load-run.txt @@ -0,0 +1,6 @@ +{"profile": "320x200", "bytes_per_image": 111817, "repeat": 1, "concurrency": 2, "requested_duration": 20, "elapsed_seconds": 20.093, "warmup_completed": 1, "completed": 80, "successful_rps": 3.982, "failures": [], "error_fraction": 0.0, "p50": 0.401, "p95": 0.7265, "p99": 0.7436, "upload_p95": 0.0781} +{"profile": "320x200", "bytes_per_image": 111817, "repeat": 2, "concurrency": 2, "requested_duration": 20, "elapsed_seconds": 20.24, "warmup_completed": 1, "completed": 73, "successful_rps": 3.607, "failures": [], "error_fraction": 0.0, "p50": 0.4219, "p95": 0.7366, "p99": 0.7857, "upload_p95": 0.0893} +{"profile": "320x200", "bytes_per_image": 111817, "repeat": 3, "concurrency": 2, "requested_duration": 20, "elapsed_seconds": 20.004, "warmup_completed": 1, "completed": 71, "successful_rps": 3.549, "failures": [], "error_fraction": 0.0, "p50": 0.4389, "p95": 0.7694, "p99": 0.82, "upload_p95": 0.089} +{"profile": "1024x768", "bytes_per_image": 2360847, "repeat": 1, "concurrency": 2, "requested_duration": 20, "elapsed_seconds": 20.567, "warmup_completed": 1, "completed": 45, "successful_rps": 2.188, "failures": [], "error_fraction": 0.0, "p50": 0.8308, "p95": 1.1487, "p99": 1.1615, "upload_p95": 0.2497} +{"profile": "1024x768", "bytes_per_image": 2360847, "repeat": 2, "concurrency": 2, "requested_duration": 20, "elapsed_seconds": 20.396, "warmup_completed": 1, "completed": 32, "successful_rps": 1.569, "failures": ["Failure: GET /api/v1/attachments/63d3337b-5ac0-478b-9e30-cde11dd2fd07: connection/TLS failure (URLError)"], "error_fraction": 0.030303030303030304, "p50": 0.79, "p95": 1.1397, "p99": 1.2381, "upload_p95": 0.1837} +{"profile": "1024x768", "bytes_per_image": 2360847, "repeat": 3, "concurrency": 2, "requested_duration": 20, "elapsed_seconds": 20.969, "warmup_completed": 1, "completed": 40, "successful_rps": 1.908, "failures": [], "error_fraction": 0.0, "p50": 1.0675, "p95": 1.2299, "p99": 1.3224, "upload_p95": 0.2471} diff --git a/evidence/load.png b/evidence/load.png new file mode 100644 index 0000000000000000000000000000000000000000..8e504b94dcf016764bba634df086a8128ca50ce6 GIT binary patch literal 47604 zcmeFZXH*nh6gAkQVtS~6iXusZh$P85C|QyO$-zKw8jzepF@S-jfPjExkkI5D6j4NS zYLKW%Zi2)nH9e>By>I55A2YMoteN>Uwbm0JeXF|a*16~Gv(G*^C=`l9 zMOj`Og(8dra<@&Mvl& z4gx%U*LXy@m~32Kom|9uc`^TE1dpS$HLqGBi!NN{n3J-h3kr2U9{KNpid)bd6sq!$ ziu`q5&v)}9M?BB;5&kSw-@LJO`^IY?Dc6V(%CGn2&ZKcXXVyN%`TWZBEAIt4y5qIe zo}ZzdWvaco^mtS?@K8ftN9CT!*qafH;%XJ)X2PoE&b-xh`+QrgeXN*E$8w98BY%H~eU^Y6|!TMd-_kr>U76YLb>civhm-)Sq_r_6P{+ZFjzVj&_6WWlWA)7G+!ja zyr0D+6F*kP?ON@bnpNg@W2QcnPRd*NDaFOh>hZ!+j}D)>`9U|E)oreKYB>j6cQs&d z%erQxql!L5BZ1b|C{rV$wPdgq-K083^`hEi>HGYT;&jy*9_>0`ujaRWscCAl+C|nK zJaB%*ug}loh1-u(P(&^NZfbGqkYYD1wh|gM{qp?mYrs{45=hY7i^MMd6pO{6<0c-tqjF&s&pZ%;#`<7dc5Ms4~q1h_|IlyQ|6_ z&mKP>*x;rJuUp3%^INr^S4ougs#z1r?s{>g{hb(#x$m+=xwT6pUPKb;B)2-w;faNq}}!TwWf zz;(=Bh;408B1bubdFnBNv`3I}=@#db`a#@X?T9b&i6ckD{7t2~gAsi(p1D1?p?&|8 zS6yA*N;7-XTTDEgYP3@izjZs;m>(hG4UehOV7a5?BjzA-ipZ{1B_*R!@>YKm(POnD zfVB53|EOb6#;Z3RI*MduWD^?;Ln}Y5lVYj+1B>g!*3Ly_7;u`^R9r~bOTT&cN3{pG zQdyqBt6mE)?PuHh17s*=ZDIQXru$8yLSgHVQ#0a1c=B$qPPJK1w;^m;fp2H#sGg?D`0GZJhB4I4vN9tJi!_!LaEG*rEh8OrP zf|v28_5LGiA)Pr71T4Q>tZd`YE_st(*Ktr_yK?2*8Udb7h`lQJiF1Ns3cIPpEI#pHS(UN%TS?vY&2YlZ2bpMtCog_%9X>VKFcX%|_j=!9MRaN`r@ z2@gh5Cs}d#`72(lQ$p+yS0M%3G%Np%~N>2Zjnkq$ZU#9ZRE*y8ML`}0ltj1r3#I!%egAqIhdgC({Z73GW8RY`Nfl0?Z~LG6tV zN;L6Q`j^JM7MBu7%xKMz`EpGeV##b~o#Ni^R=QEC-Q}zTza94w-tDak8^%t;EcgXW z2J}@WrP6z@tih$XwiyFTv2o8MH$1K__Z0Uz?2xid(>e)9{H#^zsbBBm##p$>yIq+!hBe|; z8;gmfy}9$ANLnZ#OIfIPkXWd$%de};7onH>Wq;rSv98N4Hd|c05%7&+fy@}Ravpg;9;GF zb9Dz-B(jt5-#)X^OEi=TSX(c57@?h<&ab11%5ZqK`Qu#Sok#w7I&MgsIA%ET8+2#TmC}ho{2RsZ;upnvFiESweUMkrc7c9W`axM`I!)B zvs%npom6+tDMsm%62~#=?_17Ot+pO!>zm8>Rl!Blw;N}uU1(`|asm^OB<}V-eLWMe zKHa}=)3GpA&JC0HYK3&gyspMjKlC3)>ETF8FXv~g5Xc6L?tFS=vqJy}|F&!+`WGIL zj+F_hQ^m{nC%B1KX}FDu&*3?-HTTNYbz2OIt<>M_*fxdI)AQ6@G)G25FqW9d4_CTy z+kz8u_VHcs)9=bx_1GTE(dT`JH7&C@Y#FKcpx+2+45o|*CnNsOT;}3s&BS>pe{?k% ziQ&B#@Vppo5BS=cm|}RamL5$R`pL4Qa_-!o47FG)+c>we2WzD2oyz+dfd}LAEr>gO z*5U~h#+{s;>`sWta=1&#yCnBieyHxj@2}CD_lUzS(cH#qwi5311?$VxNdfK^FLww_ z#*u8QR)gy^T`XVy465BRB}*O&4jq*)(_9A*9MEbv1`pTLm!rSXjvXR=J|$DKxw$D{ z`H#;|w!~OB){4TGe;?+<(vvJ)ld)s1C1Iya zXYBXlVoz0mN}N7n9hRd)9PfuEdp!5N<1LTTaukZy>X0lL&({yPbLJVYeN{wFY%c$f zcAf2xo%Lpx*Q}ch8RgW^H_*#9xSOq;9ht`~_w3{=K@u8UYb9a0vNfG#yfOdd{#4)4 z;z%|3{jV|v{`P`~S23<0 z!bDk}?R(`@S#EoP3XHec@z&AC1$RCfoC*}QAL!hm^}cso^PL%Z-gm=&q!w>yHO-rk zA3v6l3xGIPw)NsHr8L;ug+R79!-cK!aH%PW>&y<46p!^5;XZs&z zn0Ujz(OMtwrJB`HjK5Uj`b6_<<(5urCEPl3*rg-(eMJ7j5r!%*{N4uw_AULl+ZznGC+8!QTKhML$ zL&hYt`zL6@ZTQ#+%jU>0Tr9Ug=uXKO`D+-+99(;xO0K^qaD~}}jA_w-(ZJleyx4!y zZ>>uynz%zSb{MX>D@`&gvAK&D@Za5ZS|vYxIF^v)whbsx?Hbp7o451Yw+NQ9_cjpL zxd9XK&_nU)fBx#+H$Ky6WuKdv!E_JW}M7YQ{)$ zB(Nx*EUt5_oh|EJsJ#1`yrthf03+?Qa5z8jlaG|Gl>NZ9VvOr3K+LyGH_%S9_2$gv z#l1UA^^QA?oVr46!=D+k*iVwS!yb-WA(-r-i zT{F(lu-_wFJKcV)Z7?Qytu&V%zb;=)KIH1}XGX|#p8OuW^g_~W<*hI!L15%m9!U;D z4>v$Neo~AycFc5pkGR&O$xT5kSkQQusL@H72fX&#kMR|)(D(GoC#D^HdwWwmw|VtC ziE;;zZr6HDwO;7a3s{<&ep|+ET2=CcV{k#O;EbB=P-zgM%MRN4=u6`JDgk+xC8WLd?0dg{$ZBb{_I@ur{@XuN5f_(b z2LbdNmj!LXs(Kcm!7i4|KDH*i2N)}2s-Lsuvzhw&iw90iaoC#HrMkUmzeJZ1+(k|1 zopL+#>{Z=w@3_L0HBAh#-+N!%hi+$zP(%+%*ygbF=dxQvE_3Fy(CNL2`cdY@PMsxuAM^yxYau5ILWwxl={S&G}n)1T=cW8ot?JVO$|$-c9+7JbrU~P;(g5O57PTxo;!RA zXQ@C(ITy0@z=L3GZBQ?R#^ug-2jBM?L!7F6FX{k_^@51QaDf!A=4as~SYd|7L+{l# z5%u=PV8Q8KSdVSLpPgd-{_=uAgL?T?B(A27kI#40$%O>@R@^XY!-l~nFZNE}>hf5n zQeodS&$XEdj3LDLU_sJF$C|s&IZYWM)a-FSZR_-9*IW(aejDMFujD6-A=on$DOe{zlg!z zu_tow_%{`*E-iQ;Jv5EMm{0x$PR4sJi_5%jSaNZw9Gji|E4Y?WmNQHoKBkIxviYzu6^HyQ~J28dl`*HcE|UIMmF z^r9=&cv-6W*lx6@vgS|230Qc88rumIvu=#~u1P-A2_9hdKR>1Tm0_rP^q&j-jb{q_s5!g3f~@>7=?V?&X)&qsrQh<& zpJ*gX+8TfT`s!kvU-FG7YL^_8O5URFyQoufHwqzleBc?WsrlT7&l9hL=1#M-G)?ku zNCPBw(!9m!is-!LhD~zc4DN};Dz|Z2a88Oh1-I3QF9A`-3a%{O4bvNB*KfW^o2$~> zmU%2ChT*QrWzPWLyy%qFO&arVxF&pD8Ly(`=jT&$x+;qNdPC7hy~9Y=R9a^09-7aX zqT-7lDK+JjBw7Ycj)&W*#K~3+XKC#|f5@g;@i_vnW1%%xplO!iphox{oFud@mG-47bw{$_(l|(ma@M=hL4;zSLB`)? zyd_29g<7(gAJ(SOnc^w>x+jG@-lKagfV8lImOrap=$|;dM9D3_Xk*^(mk+?K%iq0v zMQSm`Z_o}R_+^9C1wp3VF>Bn7DO`sl#l3f;4{nUFAl_d4XQgUA3iZeNuIAoTDZ@ou z>-KkrQe~mHoa9e=m)0hl996}h?hC@yeppYBG2>^q29IHzKZASy5*I`)SbLn}w)WKQ zcD9JMNv(I^yF0gV{%@{mF@N`>Bbw7@dHwnMaKzTTU}+{z)4;pvi??%6?0V+fVQ?r-I=ku(C%cFz$oF~I@#~!9zSrTK=}eAKthF8mgDup|6ZP$Qev zgZ!jb+V?GqQq{R8mEFEmL#avndZE&27LER{GknQ-LxL*3!diz6>Fs;1Pmd0_ZD(tx zK0T&57n~cMyCgI@zelrCo54P`E}C<(|MsL97ZKsBhpUBNf<3O%X>Xuk)MBq~aE&miJHEtzT7>I`Atb$VzI~ z%2GXwYUG%JQqTHxBVyA0NJ>=-Ze*S2fi(WUwMLu)DJFP`X?$&FD$EXE(}tUgAB8^Y zl>SCe!OojZQCcFURwK3AXm@~*;Gg8#rrsC4bvF40?i(c6Gm^UPiCk-kU{QJAlW1xL zw_~N9IP39<#~RkqWueH#;kH0|~{Zlc^!Zae7_8!atEw-1W+^WPWb(&!0b)cd(p01?+Vm z_p2BchmU;DpmkZ7(4DQ@Tzx%k~S=r1e4p;!C@W6+O@1I$N7BPpq)iJ#|a! zT)Wi1sN>!6#n@nnw70V)F+w_Uki5~mUhs<+(1UIFr8qK{*0U5j#XgHWe@c)wQS%yi zVb7oS-uv_URNKtiym)@hprFg6TS1K8WVbbi@ye7Q8uH?DN|$DWr73i0ynDB~yI%Po zy+{{w^>*kUBhH9?!_H7t?%)mD;Ql>Zan{-*6D8r9jUJ<8v{|R(zjMvky=Qk%7@^5+ z2(1a3m7dRDGWB>&B#msH>|63w?{NS5@qj>)mgal3out&myq&Z?1f+hQQ^w6GVhog$ zm%VjsFD770l?J0;zNhBwP&H?WwYGONZAa)I^IHKE+p8Oci-2$2v; z20tWCW8=)H##FwPKK1v!xW~HCJ5&FGbw*)@kenS|-_@^sXv#n3quwnwyjO33 zuuybSaAPYe7MzS15(-<~dGLukNn%;0XVB(>8@H?P;;`6j|8L&+^m{D6oXe9+`E<%p zU0o>c6AMMRl9JM410CvF6%C~_%#c-1h*t8A!{ys?kkaV9F-2?7PB%jAXtT~YqnEkY zc(i)J$%tFD>Jhc zyxO=%Dh+MuqQ2;DnV#c%D3Hb5xzW2J*L!`Ahub{J+Q@g1fr83Vb7OArVwtBiaeKY3 zcJWjwnfYkzU;uYkPWHrjyu)-Ty+jgxe+h~2v&lxJ#By3X z$vO%Xu4?J%+y0T0v33)D?*DWqnA+Ngo(=DAR^wUtxuXdyx*@=2o{teOF46o)~Y`?y|py5|nN$u&p_P?2R!6$Fbyf9Li z2J8HGY*vZ*W3K9-e$GrhkdLFi@gnGI+iwi7!s9xtFb1jC`?YcUJ9pDrsMbFG%rd~r zzkEAnx+;^CXIws*f5uJz;8CSex?_aleG;S-Qnd1sGuMnnJ%N3oim!5=y|*Z&v|*j( zXI1`c>QcGcEP5nEFOEU2=SLQB&;yGhg0&xdt3ua~+p3FA1!pi8d#q>G?Y-r@-};-J zf>s%j^OxJ+UpCfedsw8fLm%U=s+6k___1AfP3qA--$e|dd|`l>PR45!V!%}^#8aF0 z=cK+aTE32*=_kCc!-{(>(qYE#tu}^}O9%$kJLsreme}=OH7@khd}URTIL{#NnuQQf zca1eOJe&zGP1LSZMh88_s~RXNSQXubyasY=OtTXx>7`%urkpJj`VzPPVYR?b?j6OZQWU?k=BRNG`iZHMxVM^)NH!pgmN+4F3Q_I2 zagB+g41R50h}%$i$9BMW(SOPNk%K0^dYa&Rct-w9WfCt@olI9;ot(=t5hvf@<$O6jsi&=n4S#vXefOQP5F_C1eRKND0|uW zTokgiFoXp$3H)X>Jd|O3q7-@yh#)SYbIQ+f>a})PJov4=iWkK1LaDDX7uZ|pg+ZZ~ z2o@!-B)^T=S!2LT^!YT_v)(;hZc>BP?iXQPbJ6{#lPyrH;s#E#d~L3;C6ddKUx1s5 z=_3^!wtD~;ho*S7QXX3ZT8xIeqG5ZIjAN|0+lR^Q%tZd4H4+ed0dyO@IvHwHdRezF zwbXd6w(>(IksFwu5-5|6lq|2Wm+k_w!)JD+YBATZ*Da3U@@5O1=71~BcQGXZAzWHI z+mAJWvrp=#&Ekl?Rc*lY-`{XEjj3`(Sa#t6;A6Nn6D31RUQV{e7)P>eIt-aX`Y%XY zdUa8R3$oX;`QgeQe*az1;(A+&5%Yu*T>Hk@=uYbo%Uvi{mCZsWszfbLu-SQjfUl)3 zUZid3W;qF}n_NO@oZynNPwr~}hubaV&*9}8XlH6*q9fQf8CsI%o^g3Bjk9*iZ%~=<)4fjnMjaYuR?he`8n8A|h z6Er-c!yT?-d~QplQmP1^X*n-orS@hgaCE+w@%HP>>aF~~zP?;` zQ6m}S^;Ju7a}#ZPUtiPWHr8xVkoKxAr$m(<3-)yw-@TA6|t< z3E!QtCsE9>z%4i1jIkbF+DGg4g`{10og>U`D% zUTzwm#X-tNzcI2sGPAgI4i(tQrQjPTdjQ2GZS&!{`m^`hbb^GUsTf?zr8sT*j!QOG z2im6dDsfPK;slP-nPUV8)Ter(ih62#Hk%+Zlt`j|cHYtgiU)udiN19WAb=$LPGS`D)N9>1(rb_u?}WJKg?(e~ueZh;vu%F`ik) ziC>9hW>YxcBx5a@@Z3Reggk&mELUMn7OMlDh~vtOdPpM1mzEeY=B}nT-bp~pjh=Do z?=^|LabtB8W-_y^W^l2S_Oem0!HH-)4AsbD757>IClw)r4-NDh78=6g7>cd_fq99j+MvV!a;FN75;;WUt(* z{CzAgMIFyFBaCbKT;<0y)!P3?Ynz1jX2zs)++A}qC-`wQV|&($4P~CO9FLE?h1O_f zC(`x0wpcLzO+JfyYDTtOi9?9L``pTUKz|3T zlERRXWum{v9yr~pYWbMt*$_d?D1YHI+(x~<{M}@28*`Uu@ixarh>1PD@=#r|CV*DO zQrdCdB?I_#hMm!MR#VQk}}9u4&{)CKPj^^3|UpU>&OBk})Q+ zJ(9Ci=-^`7q19D4yL5xNJ6PjY?1dq!&omRbQK*pEFvaWR)zqLg7`6Hscwl8R#ysv$ z2sz8WARc|qe>kO<)7T$Ks%ys0hH!B&=8~vU{cB5O=Hw^oY%~<@ZZ#XZUi?E2H_3av z^Mq(g^eb}i$SNongA8gkjClsGiaJd}&yhubib3tqL9^-rkcvznl7T-rAZZ62TAi*& zagoY8Ttb+^H*SZqDJt}W>k%1&-auxKR|6@DGJ8VHgp53`{)(ZEaLSVJi-5pohFCI24bGpGMw{y^i=@jyg{1cJ5u4xo}KRc+Y>FO~*iK9bv5SS3Nc^bD_= z2~=I#fiGY64EXQ9Wp|aY9S8?l)lzWxv)Z4}w`KM=Q+sptHK7iC382@T%j$98p!{zM z5nH?PoXfGXfy zl^fmyvK;_7$GpH+C!r>?0=g`pnx+3P#I(p3jQYl&+7U|Jws4?&2;nE5jTuM z5ZGPmVr@H6ki4)ZXxsfQP31K!z&ZYaJzp{3&4MF7C;z+5cu@ZlSzNv;XR6|>$dmNq zZwKk~j~+Y5mny9bA%;aA*rIw}tl5a9S%p(_yc9W{ckjw(qVms^5!C3t>i+tNhnTNy z7S2oi=m;}ZTP?0T!ibL-(@yhdJZXk`b!!Hi4G}*~PJ-I6H?2$C2>+SE&mVCxx6i@Y z)9m?lR6m5TTa3{~gkQUUYRr|{axhreUx!ucm@s)wcUOyt<#u#WkyF3dvE#=|rIK(W zl_f{NvCw}w8=fN8(}ZS(Hxpjs<%UXg9IJDl>*Wx*%HlHB%EPM^lgw}VoprWnh;$%P z#BtOK10w?sN+;PPqKNX1R9Y^o zx;vfQI_RypEp+vgshuP}qbah0YeY~h0P4Sk8elR?-z);^#nq*`fa~IRN^Eus@{wzX6A@VDY`sEU z0-(gVah`$2pTe*t(K8qcico_g%0qC|vc7-5r@k91cWi<3)+I<)MAi}K6)69E$~Ge4 z>lb?l40iQ^S@_0HCI809W&K;ohZR#%EV4jkuswYE5OisD&46-;T8cAjj~_q&yl6D| zJFw6?ExmgRBI2y7(Oi7<9B>v@ZJYUvVHt+it&<&8wvt@J-i#k`Qw7Ch~tRtRSL{%3S(@6 zRY0d9ISF<+XnB@XPYr71EQF=87K;Q59*n2sE!LD z@9#9{?p}`iG{3pX^pW9k9B-vDo;NS4|1uMNCd(X^MaBfN`89awZ}7@l)O&jMVATjb z3YRY8^^W(srR=f4vm#`xk~cc9YN&QAQ!$B(i&E;y~B z+F98`O^7n2xxcOBr7qm(*gVkqn4E80==qPjtc7F#@G+D0ttW^h3r6>i5|{if z>unStZSKzY^%>>p`?n=APRpT`BS? zQ~p-_=Ws{qlu|m+%iTcz?)QpqDL*W~Mhg_((SLvF$>XP-p7}yDgZeEU-w*?6MFInYI36TGLZo9m_u|Pq z^2!=h&J~A28kDn%O<9S`;rF_Z| z#j~%UpVUema$>{%eH%dqD+ASp_kug9( zdtHj|zX`a<67HVg^UodJzEP+mqFOkZLFpOegui6S-;bpMZBBTuS?%Cr2_V~M2+CX# zWGmH@q({=4k1-Lw?fRa(?VA*pT_6lW$)aQ_*4Nj6wx!`UdjnF@ z@4x{`cB;ZqrVY^gH0H(mwBP$X4~-E(BYcMl#0v5#evd_y<~SkFf4}*^JB50Tj52AD z_@$#m^Bj-K6`1U>WrV~^rKbl4!MXBnWCb>sRh%c@HDlK>JB-{^WC4I86+kDA;2VKo z;ROj%@DdMVry&e~F!g!q9wHdD9!(W6CgAe}uz|k)+ifAAl#0meJwQkj1 zGiq;1Vv;yiz-Hq;Wtqt^Fl z8mZ&J!Nd%wMgVQCXhZ`6a?0u~9Ua}mHbk80WByq{`^3ky?B9Oiz_0s-dsm+|Z6c1X z;D`#G99TY1K#zZP2Q;vcYo|N2{~8c-Wh(l=-+=ti)O889Z(0BU2gnQ{$Nytfl5l2u z1hGk`<`Z@7&g%ure@rRs*O-62c(IwTx{|YR%EQ-D2!add)jt_xAJ0_e7z)0hM5=KV zkKo71U#;heOg%KT-#lX?dmIU1Ow#sSDiFaU=wCM+k$JP1*MWD%><Z6+n{bnX6&gOJ zMdqzjT^mp4B2S($QwU<{DSsFJq`F=8mU7m@R!LoRI>gu6TX})Up2B&k8mXk!0;8tRU4lKWbp)} z|1OCNs`)9EpQkmLS^RC=!e$ix<iBbWWILbTun`m7GBST|_}ZI}REF5^1VEFfXxX%hjk z9!5ayKKO5{&6Iy=jQWR2Q>hoBea~@^bppg3Nb8ew`pqc&j)j3jIe(xrRids&38Hao z&?>mAi3J&a&4v(%D6&MJR0ww~PD{A6*&hCbJTd8S z@O;3wneVgutU@e+%!Rn^woD+LA`AmcWrJ?B?iw$bRgZj=$}|AN&JYbe=PbJ^&&->} z{aunw`TxUYya`hTH`QB=4D36Iu7vU~dflikq} z{NVJD3T8gX+6M~nTLO?mbnnNG9u4O*EcynuAWIO~7eRN&>)-mP0@vPOxh(izJKsdC zx<}Loj72tgMU`DUY@O zep+=sr1T~ z))uxh*`f-?D^@@6d}ZVf(LrXBTm_E%`FsL5l{2sJCzrgfWLs?H&u^zZWhm@=x#CQ% z?8R74xF5cGOWCI4(X`R3T#p&w3ivM6a&uEki0i^zX}>Dt6N}JM=lLk3avOM3(A~3_ zG0)H*>S}maY~A5rvJjko8S4bTB;B;yoo`Tf z!=%Pj3+bX!jD?G%G*onMv9^(h;D6fd$XdPK{-^ugIHQ) z8B#bvDf@=qaTht7`tV)a%Lrupl&|dmwmB618TxE#ce6|?i!55A)j?~@4y%(w#1EwE z2z;&dUjG2KSQ)_7(F1eE_9Iodp^L;yuN$h=NQbKfx2BREU{<*sa&ky1Qae?VqCC3k zA!1cjRE~Q@XdX1bAYk>>M#tMnn|MzlgQd~N($Z0GOMi;`|2bg zrj_WkD0V@@ohye9J_bvonD|qMSHT0Q-5MLiK96+N0xcJkW68p-UJun`Ywn z=jV0AjZ6A&Uh6J_Li9~ME{{!A)Zl|&-c?BS%G57Pd6yt8FG%W7J1}r3N<(LEb&GL{ zO$2oLe35kM&rjHrK^~*A9wE7)&o>q00VdLeuvjRQ%9PTe=+CrL{p`S7GX4F-c5GppcaD}Dm)>vv=i{K6QgA8|V>w(zg zg(w>Xs*t!zTXVOy;Emgv_yX^zZOFnOmE-m7{`nOW9QrauWpON0H%E_KW8X?Y?rcHt z_X`0Ep@QTuz?jK6Xx}}2=#VlZ7cGGvmKG#$;(mqDM9?&$E_GVpwATCfWwp0V7lrMw zeI8w3f<+K}Z&G|b73>$`jF(qePl${K9yo|J1iyjIHn~EPJfy6$S z)pE3o9%eOgb9-UV71b!t|9iw^EDD@?Tu`{viIru(R1O4)Aj~nVC7mZ+mg&;+)EE{* zq5ub4Xe6J(<#GBmesC=UG@7%A)m0Ki9F<_yNBlu1y}@j_+ad8xg(K_VxvY<$xqSG) zD{PEmK|K)-odMq=&1;|!n)$XQGU_P@QUdxh)3xKHb^CX9h!S(ie<@BApd{|r%dGD!Q*6qrI!^6&zZ{_Y>L^G9SM z)#pHZ;^8$b4MqR$y7x!`r-V@Th_t0u!N41;`$kMesKn>)vm$(_0$bnm$pL+9$pERp z$;~%cE6Pr_TJGM&9~^CGfLoy1W5NSnm%K0qtyN02*Z!oh54 zI*itEz|R=o`rU4m$5GOL-Z;dUp*qNAf%0dCo^-a&{m~jAr_q40#ST4a=giH`aY*_d zcwS}7{xirrR34HYzE|dooaxjBavTHN3A1nmubT;@XS?RW$B%H0o$&W zCqOC4nR(B$y$%yGpFgz7sI7k-GmflOO-qy29@S)WhuQGe~)}#tUQkn^?RfqEcTJteuw+ zVpEANcR(zPsm{MzT#8KolwisUU*wwuGjmO=aT}WubDg)>Y+Z_*sGF{WAEe z-w-fRtb&5NNM%y-b5k!=cr2iWwQQ0e+K_MDxH0icWHcNsuxbPWnka_jfuiD_`xhQ4 zfT$#DS_oa;QIsr-p7vf4L}LK0`q+CR1MYF4kavuFkgbcGo3N&8cyh}xfY`7}^B{lW#+8nN(o^9s8(kvmWsfTd~y`hj1e*X=;4 zzldD@;{q!9+##eD6|qTp2lgF|lAgErfV)Fl#CL&BUjflgh!AVJs zXi69?+j@`*pt10$Fpx7Nv>1!}QdjD;f9OPS)oqTS*jlV!vH*2aI}cLrKrwek&`WqM z3Lb7mcBXJ)j5hP^1D>fzamen#YGeD@*x0WUST;AI`*IQh!Gl3|IEg94)wR4@i16;$ z8Yg0~&!H<*Wb^Jih{@uG<4z)g1~trwRA-P z46wjOn5M?>tv&Oix4sA|0T7~iAs;9 zL`*XQ|fO0cTPJ0y+`Gxfco$wV9v@^1%IHlRCoa%T&L+@&cOR3}B|%kWR-} zYocB&X}Jc44KGp6P|93e7!onVrI;PfCb8|DA83q!Ib%M4!HS6vV$~=p_Hr)YRm@90iSH&yPh~e1Frt zP+EcR=a~g6cVP+X``iKl(*h|mrh^x{AQ0&q1D>x7!GikvCK1sdptuBQr;tWA1% zK#GHj{gfNfAniavc+fy-Xk3TtZ~PFlB-JOIUpEaS_x$m-Yhx2Q9v1LlkwAhzl_1YH ztu8~#wfC`Z5Y>KExuMghg`DnEm}Nnr%8brjnCI5iaetydn~p_Z*vD00E}UxSPd=pg zQ+ob7vU$eOyiH=9XaN!n!*6Xib&0uVzIc!CQ?>3$i~@8eZlBsyqOPnQoVksl)&1kN z&w+68S#N(qB0s-x5AjL<+Z#*tY{S2mlK?5zyNT=q2MN ztHEX;fE`IeXKj(^@;z&seRFbG2M~Wo1`FNQpM`WmdZJ@}f37z>R{=RW|FCQ*qYMw= zTeXsfo6%exPzQJUYYRV}9(#f9vO>OkvOipd6I$?(!A3lAFRBN^p$@BDX8$7vlZnYU zuV4Ic9U)7%&UY)tR07#4;;)gXqm-k$3@3ng=_>0H20wO(8v!{^FR)lFpkGAX}opL_HG2q_O}i504s9>|)Bhut^g zqA1`*#MuE3;6aLmKyCmQ)J&V*Hi6#V#(RazuwlSEQ-q8}4fH{eG_oU#>(b~5owa;g z2Y7lP=ba6V3)g;dxVvRAPp=}{2^B^|sMiDa@>Yuh%FiMljMBaa?u)~=CuP?@$_=qv z?!WMB*RIu7BPX&BjA3bauiXCdCVMIQN&9+AZb1_hlyjmHT>`iY3viB?F^<|(c|(Zf z491S-^BoVA^xri=1p8YM>RI8?_8;{a^-`)M?fIz0@~8h#*bRKW5X#`+m+}38aw!jTf0uz*rsDeX?5`my*d_Rtp%!D?yA`!F@SztyuoI zd|`7VMTo%Y6aJzpshzxLHQ$Vox&CQ{Vudsei4K@*pcrogJ&^F%!bK&^I>m#!ymmJW z4;?;SxWBDc#W_^hji9ddvTolR=-Yh*J4K2A`<$x0AH{)cDnU;}dI377lQ?GN-uuaz zplz;(PTE)W`}iXPtJZKRNFzK{@r(VXSWQMoW&v^p=Z$&8XXqJNt7WdW$caC-_f;h0 z)6?uoZi7hm0qfZJz%~Mx5d9;lCRLz%m=0M}JI{U+I6v2&{(|Gt9&{#uhn$jGy&g%u zQ(3y!k(wW>VWn`U>(Ogu5~02o9v*(Wx(7^x1C}-t%~iBB6wXJ5K1B?nCP9Sc52E;2 zVkZ&x5qNP^1U(@f9LV0)5RDGvZ9rua+cooF%EKi67oe6pj2M+UjyWvu@0nB5Z49YD zh<+{u9*AaT+(mBXrx+3(5fly`i!!|ct}NiSPc`I9`)z*%7iTz$j2gapfGWD(9XMAo z#1Mqsh-7=A#l@!e`#UFm%*uM~7XgouFR*0}h&C94CfefrdpXiJy&49Q6((!4=dS*1 z?xnP}^yNSM_Fc#gYMkyOOI1BV_&jjnRN!N{TXAr5VG|um=5fN9?>0z+k1AXQh!J^7 zj@;$J{?y7(xQ^G!yvje+at(0QNls&<g3!T*iAPnpa*X6_Kgk+J+SDZy{0dqS6HAL-fZMGp7h*jYbrkc-Ocf{q!f&nQZ zdrBb`+P{b*hFoUtMh;BPP-2hxV|eHoa7+VpRPM9g3IIFX{xfEINbq5Ugw3t3ABjD( z&rXUY@!fyQWl$iC^i(~%1b+0tEA|8QN&_uA%v=<6%|6OpT(nSebmy4B98oi0cQ1HiP)6l-b3>zW*i_5rCE3g4Z zGst++J3bKEFJDPH5I%Dfw$0vxA5T5{$kNXY)t0buw9@UYmOd`3ea9 z$R=wlF+2%tkleyrZtYwJW%_lXB<`SHz!SUt(JHmO{oga$CcDtPPe&LZql>-Q?Kq6f>?;l; z014%!0618EFbC<{e|du!G)|xjFoi}fd(2dO0{RgvrP32*TwvIjdxZtGk}Yv$T-UCJ z-$pPEc6y=ST<*pVwY*QzX`~S64lbw?d_N?;-(DJMey{P{C-xo?f!L#DLauU}5i}1A z^wmGC(Qic~5X;M;}WQb;| zs5DR-G)VI#<*fUO?|Z*I zGn}aM;Pa%Dq$VS2^o))10dG{expg-w}E93CzXSJb6twuV5iP@6)* zNa^12P!Ps=><{~{b3Q%*z?Fpez6(?)s9&De$*}W9UxiiEjc-lhoI8dgA*zXZ+#9pJ z3H2K7dGNO5=Hb5d-|8S%LV@`C`X8xA$l{z6o&Vcy87^z%{HwjjZ4+lK5haaf$m017 z%p|&O__F|urv=f9Y%x|!n_*<7v-;&|{z-*rrwGouC#yVRmZXIGWNu0UYHHJ7x0RAT z9^@mIo|@FSs&6wLDMti;@o!{_Mq(x@7YOXWGxH6UjP2XqFl<618udf}H!zftre`oR zxn))I)1>p~&ILxq005~EXH>V$m07)P*Q3!tfBwwTT%I<4?iB_!pX~t?i2g(T^pLS`uYTEyZb#kD&hFVas}B$7BcA)< zqQ2d?g#x_jm~5zt>1F>A1qrL3O(EAhU=*jB?lx416zVH$wJY4<=_9It0ahL^d;Bu5 z4-Q(pC|7grMhL%l0f7(WTfp#1?|lJSyF==w4`7OH#LGHh)HJ=7eSP_z?-hw#@S~Zw zuJo0vZxK&rRU!eo-C6%-hP#f|Wz7dt;E&=gNFn`;+h7!d!lCuUFo-I8xJ`tVNFh46 zNN!>iEoOgg1Xw8mN}reIh0CcKW0D4)?C(?K`T6+vatRB_+*o@;GMK)l(yQ?t55cv& z4!pZ*c_FUlv^Ir+)2~N!Psu_VzXY|{b&N4(SlsxR9C%;ngjJSB%BaznQ7x7G{CKND zUT?{7pnJyAKhptHK1WK19=c>LcfjG9b-J22mQth?nSp(AQ<_TuAr2mgN$H0+_`*XxP3fRNZegi9RnBL8tkpZW?n?!S30c#&UD7J?w`& z((m(M%-o{A7K?-T#NK^^SC7DBh~MaadI}~(HydV%t$VQzwwpRnd3ktza3WtT*g&%t z(uAOJ9r!?SCQfm;#U8pItGB2+rK3Jdr~sCm*9GaU!|DRm6xvsr^NY?h4)v<;DiMtN zCVYcu*nAAPT>~7RgT!VABLb{%enOlwX1b;U{t3(sO3scuH0R#;S-u(gLd)VADGB$i zr}jYC!Ue(Qt9VdD^YuM$Qm1FMo|QB_%LX*zP1z&d99plO%7Z!n>P0X+OuQ=lct3-r zux+UIt%-+t?USp?@>fOO`ks;w6fnIVNzBh^=NWH;DYoHJZ2iM=K`72Cb2rG`GaOHduKwl$PnCp9cQYfkegyve^3I-v0b@bqXS^>-VP-pHzqeuH=C_1of3I)zP?c0Irf|2OWn$|p?Pp~N#mRbDg78^Z# zOf!pP)-VbGR+x9rt+k1nYaemT5Cn6+?CIX5u-YPjLmDsE3#<~dDvtoTe9>Cw#ADg_ zwH_~Cys%Kch|20sxH&D4j_kRcPvK9NQtDWx`1b=C1zZOICvV)V1_WdgrH?doQB(Hm z!2%&7a5BO`I|oi>V>T(~JERb>k(+8h$TfCCp!2d1(XR1&+;zZHYs%>wxbFd$e;qV_ z(3MD;ol)ZsbuYM&6RQbMOcbExVG3-lAKXd?i8j_nCYCiFG&iMmdgi?DuC6Cf5mTZS zoVphmJ@hWy8};o7x=o9UnV4*?8}l5qmpp-uFSjfIOr0=(y!-jj^mcXuo^r6U{HJxn zLg(!vYV!f($4!F!(1$N{Vma=nG9_uSi#yuYpv~_>gwuhEwYnrb<~OrUj?mHTJnKQ3wn^G z(%`T+)~J$qL>YV>qXaP3eDhg-FQyt{N6ua}_;K$>)SBOQa3|cMNQ|?w=m&Ac@_~;1 z?fXJ3k#S}3Z##4LZ2oC4p?3ZM5&eBZS$6nP02o^pGShQ5Q9Ym9@pkSs0RiI8q!~@P zyEAHUf7F3N{wcWk?D{7W5}`v>m56lK`tLYBl_e0R562UqZUsxXqi3(kw;4JG>FdJR2`6YWT%=MEASgztV5<^qGduqlLF)KN`}QuxeAy31ZXoyrb6+Xn!9#eGSc z)g4f(WRt8x@NwgYY=po!f&x1RV2G-9qzf(W00Jp7>om(Dnmj* zf#t7KsqA?qEf|Hi?@l1s70=BH^cme-`d39)LL8>6l})V*l3ofOQk-4Q zMmYj`Sf!ge7)O2W5wsc#eE%gobBvD2B9Piy$B*BKPL$|(v8^>r(k^Spe7`K7wnWQz zTOrGn37RKoX`hamwL{+YR#>LkzMYZ!g&qQQTe7Zi=D5I)My$UO1h=l)))s4Je|T@M zTw{h4_@|}zz)hEh9J9aKsqBf%z)|}@Va!Nl2k=-rm*s zUVj9vTKxyev%o41Z^4A~Qu~gb`fGAhvGN-9LqL$!gjq%@Me+YV1g`A<36~aW&g_ve zV6FS=%^;HQ?_k_)(fo^$i_0^Y9kM}b=Z8zL4pz4geF?#H%w)UJ%yct3xJR+{()vx-eZ;9L`Py+!xx&~a_jqgdn{eOAM2UXdE`8y^n6ChU?R!%k78^( z+ehRgiulZsimPgOT0T8lv~)H-uVs|?@1eF+)W!rZjeyeHYSoMYopgj59Q5s{H*_+T zhyavbg}<>dy_-eVJ4W;Uh$n^VT__NQh5?}1S-k6iu|dK50U1$8XBYzFP4WbBHOzzk zj*D$D+6=3{2&nTXjQ4#ll%P1=YBy_Lrn%D5`Y`@i1h>fz>mbE-rT)ctLY_? zSJkblAfI6z9cusW9buyU{o^&g)*-Lb%)Sq1m7fh6{H5wVPL}?Qs#{FP0lB$Dc=No6 zj)GD#vNJ%wEho}`i{gMX1vB-1{}o(JW$*hKv*!zg!Lt)Aq#OiK598v7>&M=MWbEa1 zbv+0m0;r?k*ko|l4nrF#&u1i>Y_SsboGrCcp$WIV%(3aR`nr&P5X;@v0-ci?EPM~h zxFU=y{A2{~!~99U@^o%bo{o9`Gsok_Y!GR4))l5XpTL}WY?9^Gjg!46&6Ud=qvqg$ zAqDb+h0wlVqiNkjVFbYwR*pe(&oI$WzOyUOxS>-gS-7rh2iT`oB08Vl2CWkZnY#b1 zh!VXKJHfJb!(2@_|#CJX1 z;JxaL%gW=wna1Oiujl2}_-gU3Fl|^4Cr^G`6tdfru*@AJdo;htD#nb!3mN-aB$w(e zR23#1Pus+3$j1$7lQ4DQ%&Ln0)uRblNLugQ8#(8~ZFDYBE5>EpGjxPhA^1LBQ6sB^ ztM3)ZKYW(ZBL-|}Y5ii#6ntN{RB>UQf$*j)x2f61r_i*r9Zy!QMfuo8!0&wU%!%A99mqa`b&lD-wPvS3!C zRV`xLE+Ub?FQOQs(VC0>^90JqIYqP(*l*+C{__!nAPbvb8rrLT)!EVE17XRN5;~A* zUK1;eW-PGA3KN{0!ymkwvA%R08>p;9+C{vc5Oda&aGR$U&|pLuIwY_ zf>RP?_Jy8PkTo%X2OtD1xP<1*|j>Blc8g_x% zxPj*U88brm3)COD+3A)aA6)}l_Cf)*o~vW+)m2)_dtINO?D*R8!I|Fgh&>EQO&sfI ztSs7eRsj9!lNr^yev7<5MD8yBTPM!vY3T`oQgRA|7s6L}_wG`=LnO{cY7od9OwhK!{z zgt8kD4H$@pZo30?8c)hG!bAu^g2?&)m$kP23^_s2=xGbF+OENcd-Y%52SJRSr!`UB zQ0Okg#s|!~#kW4%RJ5>!Q8E;aQ4&%($wm{IFcfuvga6Epyb-8z2xR$-nKpG?ghdh` zR2%78PYM*RxPRgk|D z@bjmhyXTMo-w~s-X2^tbC)!LX{ahDg}}mBAP;wtUIdA=^Z-qGVkqw zTcyB9@~H5kZIFWNj#(Gp)z`PGZ;ga7p3#2%^RL03R#S#CW-oJ5y={v z7BQT6nvbVeX?N9{VmFip2U7T z%A0rs7`WfEri{#zibK<@RshFVHP`^p$G2a3Y#ab*iG(z%2T0s_@`Mh}(ItfA2bnnq?MNkzGD>KlKGusrs(%ym8^C*v=nTeZ zW==aYFKIMw4g=wk29TtBv?qMvU=JS9^!+t7l<&HC?_QS^1l_1TPT$kBK%%vnRH}#d zk{v8j=y*Z-ve8%Cwu+(VT!xMxa~hLHh9S`}4xtXA)ChP_1Xu0JS^?#*HuA-|{e!S| zj;I-~dJ*bUi;8R5rv!kG&>~m5_TF}@1zCls7X&%?0htw>T!J#O6&9#t$MbOTYg}PS zu=3Tq8>tXM!@v6z{nTh8bA2(fu@H==_jMB|bt07z3he)C8Ue?@_1O3YodfnadUBks z|5Zw2_AE!fjx?rYfcO#9xw)8%P9z>ReCk`b=v=dmIzKzNsiPLt5|D7M1Y{+y|M;I9 z$05)o{cn<*rt01F!5=A+%$*fIW5X8ktEj<(E0cBp>1X2rBbsZ-F8J%-%cS%+)@V_K z3p~{KgJTyF39PTRiB@$J8m1rsYhv9AN8DmA=HvJ&HMnk)5Gu@o8xw!~`(S88a7I)i z9NE}TtB~U?{usE+e4v#P;>v+bb}l)1kXf^dGB%ZEl~91!;W}XLZ9wQxh5wp@mvA=V$}vyM;dZXj0Ud|Sqs6M zICc-Xiu*o8;WrQv*cWW71Jv2DF$xP^6Q8AQHD1ShH8S<~Vku&bHn_nM^hL)X3?daH z<1R$KNLxjpc5Q)mg-Z$rr{>^T2eO4X#CP z(J+kUyggE1s-g=zA9zU~+r@m~Z(M9tu&YjcV7;FoB(*H->qc;W1UDw6YhGJoF8Ry= zyBIBFQJ??m@xHT|M18W2p0|2Em*}H(CSb_{sL*u#)uEc?G7wjncS9E@1`;3vt`X{l zfN91hN=-e8^gy2$dTsR%7OiX6S6x(!=2-670YZpUg8HQ%ObKVv{tQ2^d5$Lw2N#qN zhZzM@%?R6Ajnwc+6wUz?`YNF?6(?vF)#VKz;e{Ih-arjBF)LLYb%)*{m49SLm`4LP z4N3H2m3H{5$j_kP(Sto|WEk*8#4Q*`D}p|v;qef#6b-Q-kr?YKc>-@8ar$7L6yCm< z4$^Lb@3T^+;4C1;FHjrMEmF}!6Vj^w^ddQt9>G-Wvu=7puHa4D17f@N% zPI(fiapz8O(^6hf-|wYPY5%&#g%MBUU9ljveKyiF?Jf!#S{&&@Y|sM$+O$GDK<8BJ z(^7pVFyD+A6$cE4CMa4HfVOqHz%<GgCp4vlC>0*0Pz_E!b?Veb z>&}M!r3k@)WP8Tgh=q233lzT;&d8sZ14~Nd^v8cTj9&R674TUY%SoBUAGl$H z<&e_MTIq7z%OFaL+2i%9UO3tfKB#hNJJY|CSAN21Lga!?+V}=w_^Z>Zc@|ahsxOktb4Nw z)S1R*J2dnt`~)0e7GtRdkR=i9LWRE=cOLrJz(@CZ@5i@Ydw1J2xv`9Lsrc@=5cQXa z2#w-UYEECgRSC9~g5gn~I(K9&r_CKnq9RJB6<8jmM-Gd2_mq)`xx$=YLTHG>#+=kn zM7(+T&8N&>!7eWs#TbS!u)vg0&e5qjsY0LpdO{PUsQR9)r9FQn`{v-(a1MgaJHL}a54H^#n+DNb!132aznCzn zKYBdv7!Uh(fXf-F8?$Cqai-uVyW?ZYOG3ZBLUEA293S2JBAlCGW>D?YT`K z!u=c-U_O~>!qgFT6w!{UY-C$LDrCkNZalXwG_WX0FjoDF@FPH~Zu zcTJg7Gj%6M&d2jb@lFel92J0!m3i_r)Y6Py1^}oP;wdtL0AKjYK0=~vF^fU*bmgx=Sp_pp61YsEs%1_wta`j;BtO~o8qb;V)8>d0tzsq25v_K{3x zR3+-xU0EfW{uOS&PC7P)Ue{vXLtr3@C*9kQ*bsD`+W=veYITExvAyyLm;Nk zE(3I|$`gPlXo@PyvWomzSnX<49U@Q^2Umc>sz=Ecw)fu5b#PKiJXwxpEP)|#sh3`0 zEpEKfJpBGeKI*4@_kgnTkLksBRS-Vr9eD|X3k- z6P3@7HJBwTcSe|?e}9{~qS;GN}L}x)hb{2M?vvAdXXUCQ5Xv;t_O6(u;*XIFrPWM(gcSQQ=j@8D9XH zWaf#@W-dUD8jc{>)`ORhtB^z*m&SN!#TSD+%DiV5jFWG296y!3`{|(I{?jmRVIh5?lEJ|1qVR6*#yIH0=JPSP0OdTnEVsq^(Ncm z{5WohG43YDGiGRu$f7|{X$te5fYfa-JbU)+kumIP-`n)GQzB0DPUa{RWY(}?QIu|= zzhlct|J9Nhp?XoFGWnP$*&i&G-9TSPblHXt8_2&9gDZbVP;?%=aKZ%o>!Bf3u-RDT ztqC84gmK|~5%Qbx?;VT=m}6mSoep)oYK2Y(%4I)U3eGPESXrQLti4=aT=0jdrkJS!LBq#kKGp`rorbPXRu4JxtS_vv%pj2VSMHOD?i+d;;C z=8SnDZeCZbQY7|y72;W$f~TP2>gkIZCzo@5`SO=9Uy@ZI$(W`~=)c#(MrAxE@0HP)vUu>bMC~4FV|9p=vQlZs1_Xs!!PvT#6i~|pzZto zAgm55?EXW+K^iIhb`+mATBM^yGZL>)gU&JO!^t!TAprqXl=W&e^ns}%T|l)$7)`UC6RJoEnwnL zXM~qPog)o7L03Pi1TcM*Ty+9piDffJNoHEttBK3A5U@z>z--J~$lfQ5f#kXjY$O=_ zSL%**WhR+?`5l5{2(q|Ml!|s#=&{Ws?vAD1dIA~m&wPJp)E?TS(S1_A*zhAp0*kuA zHxT$$4=It665Z|@Cs@ic7n0qEl8G0p3rCQ!4Ur=#2B)fg$~^j1>;%Vqku~V1^Ig$S z(^UTP3TO+Zj4B?I4v%C`+0zvhRaFBXqUcVpw51mbjpqcUNqFV-RROcwq8{bJ%x~A9mXw#kq8<`Rv7ST0fLO z94u=!(|RokVoFC7jVTa$e}I%)vGNN-!R79oAJItFqjtLnp@A;Ah}M!8gBCkfy(RnR zF7u4ODgT*hex4jyb$$uy0Kl#HtjXYs@&oylRQERvS_>x@#y`)(lD%m{B2PZy?lWvv zk);aLXhu#6V|Ezzbdrza43dt9CaGNKYC$~jcy#R+RyyK{7tI?g z4%G46+m(Vjr(6NU4`o>MSvN0Igic6j3gMKTkgO(_yUDiDBuf6o)YeO*H)iAU7w*(CE0jn!+p5;9iecpMTyXKXxdhGr-BC|YCt(B zcJ8!k3-Ep`^;TFK;+ZV>aApu~@gaDR!t7tZL@%U7dIx4L?91M+$uE~%v!;AM@>auY zY?4L#qL9L(ZIU>T34*JVf}a!3$_XAt2tqFKusq%qXg_#{y6KD+LHyfF>>dd=BD z)J4XW2bnh-KWak7p29$E!@nGLv$jA^X}B4*=o#TP-7esXVn6JpucfXIF#ur_M>-9H z6lqJihFH&&UKUWT92kRv(mNGR2_I1Y=1c8B*FZ{Wo8|N_N@j{P)4CU5AnyWTTYi`Y z2s+eC6K0!=vB|=$VwHs&C6y`L>c3HUm^$`fp-Nphv_~~kWSzqf^0}l5tgy!e;FGOe zY)~+iFiAfbAG zJhL$8(++OxZEG<$K_OQG8;JWtCCFd)4B?puP%y?k+vYZIb=qEmC>@`G2QRN;I8_6b zv9?AO=FD^88<{uOT)?c5bL9bJ+s0aOC3rj2D|oX!pm)Hwnftfv(ssojE~OF0FkzM; zm=Z$V@+NZtNg6cBNtT{e(D}bw!%9c74Cz!<1R?(~u58~s|4;}Yoy`B8$A7N7*Rs%F zZvg&6xS5S_6Dh(x*t@V7=%h*ewvw40^JvduW#QC+O<45zCoG|jBLE})z}sOnhyGMD zx;JG${^XqGaf_?Pn_g57r14ZJm#1f{b01?)VTK`#21>voar3iyoH=AC z{38O{2y-~sVY}cQ??+Fcw%+;JctrT6XbF8FLe)TW4kg!;0X#1kQXDhb7q_gb+SNru$8%*H_&D6FyU}`r7|6j;#Iq(8e||uz zvSFVmMwwL$iOqM3Hcyz03JHrKw(04!!qivZ8$IBMSK@Ak)uv(6mz0zL<{Px(8~|-} zE;xsUn0dMk4KY@ZA3%m6IouCj4(n+lib@d)h7C0~SLmqT5tcvt7Z0leK~L&bYQx|@ zSh{cStuSl288n46x)8N{B!>j%hjx(mP+%euIK+FKwx2n3=AQKl)HKbf=w#X%5*%i? zACUf(3J>gfIz}C1wr?4zna{KMx;jRZ%I6u8xOE?A(}N*^QAN<%{G~&CU^Y&u71g)c z$zHDfC%mXma%_M1uf}47~SK7Z(FA~W| zNE8_rkr>@~$z@(NCjYUx@B==Xg~}9D#fvIs*7=hjMJsT{2CBaQXY5Om6^@Du%9$*} zi%zSeu?SH*&6N-lg9`^8@cS$ zcy9OOaf%K>pDD8|=}SF2>;_ra0OSzNTBh-h+ipKCQxXww-;5Xw@MWLl;m>RILj95j zAeO{yV(;&YY=E+12?j)^TFF>K-mz2EiUJBvKz4IBws4G|*}bg!sJF|Axh-B80~I3_ zgsduv@PSr#7Q^1>MWHSuaEfNTsnG$zRiHsFJzp0qiw4j!C3PLL=v~ZKcdLPJwSE7N zU$gi`$QBJq^NniNvWZcz!cv`Dq7u#e+iQ@E`EVg6t?0fYO9Bdxb-CBK(1If~wm-a~ z=KqPaF9?*n-#P$XDjO?VKbPLbC}I?eW>F5$|KsODOaP{84-KwGN7?|DT5U}^;kjsO zvv$#~hgnRTfNFTbH1=ONBDK7pv5F?H)lbPn15;s7NE&jw24LZ`#mu`*98}7e;aM0xFHeDy6YOKAcHi#`{1 zLcTV(wb63w{XVB4jfrX2E)r`lRI0D}WyC8tjQjZEtmfMNQ+D_+`P*(Dj0xZ1)nurX z`^xy+!o{K=dR+{@)D`jEE$L(NkryEisEXcrnX)Vm;$En$W1rcST> zU!rw*C%^xohjicz{r!#9lx>$bXpI$Gapbl7z=2QmV-w3|XOpLzv{lRQrqYM!9Kt(f zTBtNfPVqlzIeFNKa{EZm_R^Cz8VoK>l0l5<})S-aLuJ~Hc9|4-h*mtKz@ zlHor{AB-NN$5Dt8;eb5ghw~Ubzkw>Es@pHa5I-M>ZhCeFid7U zHLMN@6I`iXfpj7AJcX)0_T6zx#QpewnAjJO0mA>X?wj!m(1wO{n`itFCr#m z*H;f^fWp|kDNY#-z8Er)bRxvxIp`Qj*wG0c<{Y`h`YtC(_rtlaUUT?0;#iKVu&XFq zWnbhr+SG)$xb&5{F*N(K&Zh4?6Q&%qciYUt3tpsHV>&6MrnNHZ$!j^3UB2kAauLF7 z;n*3@p=vjED1#WeiDTrB>ZQB6lFOgM$#x~+=VInL{yylYgAI}5>(Q{v z6jQXNt&?g~2=ZX1YI!W(2ZYL)9H?l}A8RN08{t|PLVy9PKw}{=cj(+`c$HC#n++O$-(VU!}bl1h|^RaP+BFqLjMmjdaLDi`(u@ll`EuD ze@B`nLL8KDYEKM7OTC+hA4E^}T!Xx9_aeZh)AT6vtW3D+1B=qvGd7q0kM-W?&cTl} zz}YsOkqDh-vJm*j^OU-s({dPK>Q`-JVnp#MZ>aCp0l8~GHx_ilp1lXf;e~hOI8@6w zu!k6FOR?ld(7^{RP`NDnDQfl|ZlMh?z3R)ZIX9S044swj<9X&AKDYa{Meq2z6GZUW zX%l|0UUTt>!-Pn5;lb6JFy#t?^v@@(4?fG2w$LtYbr5TlP}?av#_r6xbvgC<8Ocl_ z$IC@6b-Y-_-cpeP5$SaE6jXnK8_$vYCFJtb7(54d3b}2iq%N*^lwH^cFh&yAM@+}S zetDHebti3*SAHBC4ARw{7^D!(zEoJ)Aduz0;l@ToXD&8=!ZMn|LGS#(t}<*fnOh7y zk>SV+A$^(KkkcmsEMItC}?^vH1H77Y}QM81(V)AH2zGvu`Z zRN-K7m={vXGD3uatlg4*xZKU)Rvi(EjIgN&Mr zJZ?c0!z)+0{vtbRhzR^#Msytq(GoJQt7zh{V;)iipm5%PR~wK&kv8<1et2O@6r#k2 zFk0d^{k(c6c!!ZfkI2klu;XsHx769e!3l$hW@V4}{YP$+>IRfrpq&f`lcrMp&#Jb> z0F2D(tK`E)5(?1!!Dj#c(dkPm*jJi1VSsExs6{_gqEcGlD=5!{mimFdELMYm66O}SY3gBn_^d}rZ)yQ=hN3Yg^S2Q3YkU5*~h zL)ZC*Aeclytc+}Y<+jK9kdtTyLoDGxVr!8kzy=-SO`{)L^?5gnr!Szq$T|T_6*Y(@ z>ixDT`jFyb>bH?T;MN-R?QN)QbxD0k*eRCY#o%;^#tzf#*gW>LS@70_*!7>;%eTT% zCzXC&?TNNBnC_^NF6MG_7F;>)!qKH?tGx`n;>n z$zrB++*hC3GvU~8pV~RmZqdg)g5uE} z_G9*$Gmq7LLfzF=-&`2Kgsoyc@t=RrY#%fC%=YP1Yt*J%F2L@q(>H|pJ$^Q30Uwv> zv>(5JUHI)Bma1wZkexQ*_AdMR^|dSe-lg#N<<*@y=td4|N%aAS8DGBqPVQ!C{h|Kd zw*!ue+QZR*4#w`t(AL-A-{EQD??nq7WN9$`Q|}?nF^Z}A?t~ZE0}MHu(BN9{5S;zf zL8oZ~I}9OIw(-7~22|G4$9^DfF>M!`<%XMdE~(3dr(?eIlC{d4HwTqAHZ~rXr%uS* zuQR8A=eHSi)SXOFdd|U~r0D5ynl(8#GEOy}Ic2g{EcYv7tM)(@L?$q%rlxtZh=@xS zO&lgMX`(C?u7hzH+oAc4fWd+cx*L!C>b;G@jtp8STw;o^BW-IQJ{&nvb?vMjuAc9g z`fF{pC?6ZM>YNNxjz@vK&<4(Pm}^FM(@evle~k6+yI_ik$o@O;1^`F|DqU9GnE;ZI zDpz?(&c_n@fll&mr|_O7ZXg z>!=_`dwX5XXlbz*9BbbMDTp?Twr^cRFC~c?b~N~Ynp?<@d;Jb5u+1P% z5?euIZhl#UcnhV^VHJCt9@$NHc?ejd;W$$m)k!%@vwUwOrUjKge7N{d^CB^^93b6! zgGfpt;-w1gpzp>IYA!Y$WLW)gq~(k3E}6%m4SNS!?qVu>Xg+(whiLZ6xD#j_v>>D7w(>ecBjTP64o!CN#hli&_{{iBUcr(4=@kL@H z07%-W%*8)hoq=bL2ba+-2sTJvN}UKw+z$_qA31WQK^D}y)h!r)bvYp-Os{)P05c_`EBO={4S0 z$mvPmq0yTpu5A=1IP3H{kkaVWa5c%k@Quhm&BdmrymyJB;%BQ{RNj?heDSBlJ1lPp zy^H=5yNkXl((#fO#zYRUKK@-l`ItnmbnQsWkrD%2A1l<4x^(%n#0ti3Fo3yY{o63uG3m~ItauvzbXWy@sQtu^~1yh0(q#F^xR6!`|Ib+URgJ9(zyUSY>+~w4s8^ z0Q!e|%uK9mTMG^PGKT&4CbUZ@(KEUqyVBUKieOIWi{qo1Q2W+Pxb1Uq&h_d{bYS^g z(5q0YQRmkx%xIzsSOy$A@x|J6&ojFXLQakI)ipGvGvUhMv;PjkotzU0F4au0Xm^}d z2k%*55aB_6xH#8f&Xu`84byD7psbsQhwu)x7cVlSR#0;C(~p2oC?NO%IL;9~Q8|C# zzI_xEg{?auZoNCH%Fwm(5UMD;Z5bCI^n2SM?GPuyqC#hu2m!FqRp~fueM*~b z8y%01CGeP4cj3WpCk?bWqDa&JP_e1m**%7Z=dKI9!jceuN)7P9Q{a1X5`esA7}WBE z^d_*Y@_2={SY{1YA(?kJ9K^+KaH7M-?KwSf^&V*PhF%rq9nyimiuAtM}2$xVfo zvF|ItWG-*!u=mx$EjRRrn9%f-I+w0BSZU;5HFy%$4o&tq(50(KQ=wR;V=RBqp+^lB zhpzuqaP*`P`fOVse<_AK(X3I2PuKIM4=vYG{N$iKvsFQ|pt!buV$faDrQtd5ZVDeh< zRtstE+!(iwJaB0NCWvjM58OU$7pLszIN7%n6~Q*^a;Ke08`fUB=i+UOH@!zNN#13E zd>wPlVXRtba#LYcOoC0-p`)ir@(cp@tInBdN2TsQOx~%*n!br^(#6YIYMw0j{KPbY z6osK`)uRUKyPs}Ncug_L_tKRsD`TJ_FloCYrm6>$s;lX9QxdpKW|Jcxez8C$cP5u3 zl24mD)#q;8d}-+-t54{-L=_Yi@(=OjDC`*GpQbPN?wY$Jc5J{^%XhrSpM0hooazm` zFaL2NeS&PRX~{YJQ85tROEHvi=Ew<2mk(|#gmjHt?Uj1vHovnY{Fod-yDrHj z&R?=5r=`{IT2iwKqXYqV-r$Prq1b;GrWJpr^_g_Jp$l`NV4yUG%`(;P;8EXw_>_Fb zPdcw<#sHV|ThJDziOHi+NpXw%P@7sOZsgqxga+Oz18!bxvRg zp?qzHLk)Gqac6`kNkObEvs8&je4yoS`uwaeJ3sO8r?0W^2=6BHp??A1jWbV)l@a#n z{6)2`N)2b&PP3deDI~V#wgeg|=?zywwvu`fo#AGz%>_7)PGJLXT8g>othWaZ4$vGb zG#ln{ejz@`{vCpI@Rw77n{^-Hloeqjf#F2i+>g03-u{${UhL*J$LfBM^QvfXT_!Id zwg#?nzy`Dek$>ErSo;$XBx<6M%-MsIL ztISC=D(F~xbjlZkD!wo1T$8_{j4QXn=%q(&ZePyRjk|wt{bX;=?9+xPpkU$VVo%{Y zBNH)m$&*!r*RZ%XT}-!MKKCuC3;DAsgK1UhiFMp#l;&+bJNN$D&k`c#AH0Va6==G= zGWToz_4e`JX}75wrAH^WS(i~{Vt{O92vF`@29{ePO6IN=fPIsLYJ^%$i; z&th?~Z|Q%Y{Uv{0Q2~N@B&n4e?K3qdX2{F<kSI#>;K0DjN@nRq*@oeO>sy3@T1axp4hd!IDcPTU1uHh+);AB zb^P;$C-PkEzdv97v4l4|`g<0irdh>B(-Z&wRCf2yb?NcnKKM4eAUYvY>R+$emKS5m ze|u1);>~ydwIVFK)9JzCUzlMYZP~Pd#74$20z&k)ic0 z1LN-=BX|!TL}&VJiR~$d4do?CzYjUj{ntx$WnJ`@e@T~RZ3)?MDt7I^-eIS{wol3a zl_i^7tkwqq>lJ>oZW}{-A`~OXbfg$;X<4iEvhoML2lL3jf!_{OSsK zq-&BIe!aJjnfJK!_4)sPqJa%K{BeF|gXa>{o@^17(h+4Vi^eCn^51se_@DK{`6aqS zU$k@Aku^s-f8Kz_${d+MVg8M=My3v-f7s3Y?}vYJWITs?0{!p*#~0i@lT`Je8^KO+ z;mui>26l?h*AZQTFMC04@8jC%e>O~`4_e2)sPFROA*H$p0}C#F`@Kep4;L+1?e#^K zSBEa^J|4@I1@uIn#3n%>%d0901!{A!BSnf(>Cvcfd?S&Pe*Yp{E={B}f3ffqgHs=^ zew`~5T&hGrr-xl#=eMdn!K7`|s^|^H2-7p5AdY_l0q-oBn4ZY~(s-T%*U(ne-b=SeHU*-0cyVdk*SHczsVlAabN>D86{1$HqvOgH zug}P9ox*m0v&bmm%kZp4$ZgZ6H!lD9Y7?bm26DkcI&ZxfSTO44jf!!a73X)&T3frk zcFNriUR&e@eN@(ZA-4Grp4`ewcX%{wzN>(q%*(%FMM-fgE%o56-Z^t@4ZjUXv|4*C zdpxiD_2i2oJrynH?;7@tpAptLRa$Eh-9w+29s_iaGnSvd`b$}$dsd1F%hX=r!)yd) z{gS6GmPsWdzKEaUxM$_>HK@8J5T#LtMt0w`p-F@TYOWv zDe9M+c1yu$2bl*92So3irJ3ON`@_KN%s6Ni^5*2Jlwygg^n4-dK4zXzNWU}u6w zalw|Nqn|$WJZkoH+(^jM9zBAZ=o>=X0`sISBDdy4E zw+3UFh$mO_;D|~`SFZlts4po;&fjciTp#gt_HxVq$HV5U|8*U@tRLUDGF-Wd;_K)? zN30vYPgcdiD<7|qX+127T^4#_IYlNUVEz~(5u_jFF$*v(|HPHzg@I(ev`wrulS%@4+PfF-yV8CD}h(# zxT~1&Nk#Kl_W1*MD@^3~)~nv;Qn;JI%J z6FA*%ccfoxoaS;oo*1ow?e7x(m)@?Z`tw3o-A-H-pTx_>QyKXc^J7oHgL<4gf0S}L z?o&n3e3sPxdcQ0I+>a2zlI+&#zpIgR*td;~P2Ju4Y)iHo+C~6RcZ^z?5}OW{r8pqQ zB2N}SwKUj4Z3k#t|(OxK2jhf}<@{9h%1WhAr9L}xO#n)J4(Q^~2V z3udXIek-151sAd1lcnKj3PJ5<>V;_^N;o}dU1OjOdS}jeA6sne%Xn^rNEHEO+71p5$q}0{qIDx`h$@n!8=-}K7E7<2ck#x%-`q=Hy?RwNnE{mPhb(6{Yrz6_ zb#?MFwuh5Ho$j%Dy}I$WKhTY&lQq$-{Vmk4mk9xaO>Dw(|L|2U_g;O@P%7{oxrk)F&v zJYP@np2r8-!Dqd`|fcn9mnbA zzrlNSx8LahuS(EAUzYhI_#fwc#)QAA=l^X9TsO$dqFtiV$QH%ySx@#TkgIQn@hy1~ zHhoS7$gK&XG&fIX)l`uf17ohG6^iJY?Ve)|X@D&wGJe|3nPRw5())eERXfeIH-@1bS=A4~ z0d^Tw5-DVH7GEf30?|s|fJAO`gX{@R%YiPr*4}Q{x4HoiYW_eW4^(_BvUi1_8ZEaZ z_|)Ye!z8=u+kfzVXcWQdiL0sUJ*a3uS6)X8xH2HM>)qP=E6bm&^_4@^wBp#)s|g?N zQ=#r5@jH%|yCv{!NCqvF)uXz+71r>^b|!)GZ8XZ`J^L+A?GVm^~>W0rnAz zak=>Zk+_adLj>?79l#x{knkI>Ays!)BGxsdkv)dn*#HBH@wDFxR2QP!!C;mQfk1>^ zO=tyD{}GoUf>f~jfTW{fwysd2D1sB70H`hSNRjrcDl<~lHDd&5PRewcDtj!$u3Lu$ z*Sl33gTo&BA+;BfRn)2Bt~)d?8j1OeXI|i=K5M9{wVzA_SQprFxujSQRK=a0@Zsok zK6w(GK^|{JFp@uruoy|iS;s-FwkR+k_6mT!NnV|q0Polm>%^)9%`=vpR5nA$F>o{5 z%~{}nsL*mNW9)a*GtWsYGai--Spfyt@2ucfza)BWKul?XxqcCdI z7Itf#vP;vG62aAWdlR^oqU(Vc?!pSslkIT}?ZqWB1i0EDT#pPMgBb&Jj-va#dN&na*W`C#Ozg(#IL~F>8>U|e zq7itcymZegYvboG@+!fA$ZxzjlR#Ss6yMYxEe=}@sd`+~+k?+;6R_~(AGPKq+YTsT zqxUdgz(eAmEoIx+doAVg&z;+HtaB@Ff$6->;FhW;Oqn8Qj#R2%Yy-q8Osj=d)viVb zcfV=a7I75>p6&Of^Y?gS!~9ssqs$2mZ(4G(8@5vMw@RA6q@N0k!%Y$RpflcR^pZJ_ z&DlNY_U+i=U+U!KX1uHrHj3m|S}Mzu#<+#mI1!_lFuHU7_|3jqc!ANKL8x0Tb9%hE zqaVo1w&M=*p3zbobAo$MA|z zpCf$8vN!2-&37*fY3L4P#3iV1J{H3Q+t>rnx5@-73s-9NE@f|i-t%>XBluy1Q2CcKb& z9^953Z$RVgWkHeaFl}HQ;O{RAia$LgBO{9>G*Ij*_H4`C8ZFuwKG>9;kf?a*l*9Lj zzFu>TTjRC-k_Uedcn<}hUBQjJt@n*Wh+Fi#P0@;6+;;>OUCL6Mb)A1+blYNd>fwox zJ#R2Nk!OX8VA}l5G6zGDiy^_d4sLxX7G^`S{O=C0Q?#=ShJGoM(Y@2XTkjk($_?T# z;4PS;U>tax)X^eewspMH9zK1{rk15id)9Xnyj%671j}`*Rc7Io_Hd0VJ z{vPVfZx$Hh{@&JRJ3PuNV@ts*h+OU{uEg2N1;1d!P^sxxz*$jm+gIf4)%}T1GzU5 z=E$!&^6f`%RbPAK3Pa(YW@aI~H{49U^E^H`@7b(5ne)yu;eZ$6)Jh26=4Ty>BB8}j zEh*eN8PV8Y@s>UE`vK?CL-*5l&gd4aJCn6q(TT~*&jPdtzCxQK?PpPSyawEX+=l7v z!j^984vJn}@wBn{Qn{*a#QF~?Rr6aQ>8o72>L&#CbkFnuTy*>7M%q|NtREgB+LsfO zjn58m*f?77=P5pcZo8zWFjy3=ZSyB8@)rq8KR^dqhls0lWTgYo({VE$JibmeK7aRY zbNW!LEo}f{(%Nf-F6c!h4kGDGpM`OhO(-6la zI(0HHZ$UtZm5ztAk9^*Sz_Td42KNg2CN|7JeEM20zVQBpOWW7fd|a2e-I*qbX>8=m zL(<##3nafeBPm|QQN>-FKWvfwl^ntp`?lS&`;aPpyVux$th!Of^2PvzWbNzGW`!Sx zph}b;+~1 zLd})m7wyCz0Vz}Egx;y|GDa?NmVDl(&Y6f7b$_^C!CRb1-Mf}qY^~k%`{(r={#Gxl zK_#l|s&+Xv@}H)WNyu6>4m9eXgIShb8=5Vvgy2qxsBYgoR&0@;-kp-!iKFeuI&8Pi z3^kNrY|QhO8K5+&`evXP-k7oHj=_hPco?RV&cfH}kg!YvvX0ULuF3f9QnkX9Igl3! z7lF3*JD4}ucwuMTsoxcmuikUZD=g&QqBz_kC#Pj=q>o0AdNUrE#$rvl9G(n{N=Pld zxcNtNizK5|LCG@4Qo-6f=u?N)aZhI-*}oTapU^blMGf=EC`o5syR+%H-5r;Tvvv3G z-P``5N7?V36?V^}vVbT>FlH45DitEK`l}b&Xee&xB&O^os z+I@MqeRIejW2+e#TEtbVZs!`t9o4&qd+eDP$fL<_7B|MJD?B|SxJ76+)@s%yLA}Na ze1b97NXPpHC##W977Q!MqeJYVdt&wZTrj-SvXUQOWa0*^N^h%7Dvso67vLrj#n7^t zd_u!+Lv(Yc%bpmO_LivDYh>I`y%XIRUEuxc$nrQ(dG~`%4lrK9o1Nx3ew6p5B9|G- z3pO|g%cOXw>{18`b6(e?Y%qFp^7VMJ@msuXD%!yXhxUUSy^>OSOzX}VtybudR$eMN zX08IYOP*p0uVVi{Tkhy3h7_^1Xh=k~Vy^bPXvsa#zaMLp)y})&;+0Ux-(jWFV&v@5 zlAw>HI%_5dosVt-oJ9ayL#$E&k6L@%HCrn?wHwyM*YcMUo6PQA=4Ir5&llvHZ4f_FCpriRt z*ySx2bH29C&P1HJsWiV(xpp3y0ER8VikFD8)<53 zdCZtS+v4ra_1N*`avXwRa4mFHB?y2n$v#jAth8pT9|=jxcT1o(7T<$8;-F55O@faY zLF^I<;>Qqu{S%-Hmd)6Ksm$s($m#>S-=b4~#9`A7ZyVwL`}gM`ERqA;rllKQYqAVt zf^9)@sK^83r1rlJ5wC#dka+Q6ljS__FH(Q&Wr2qIbM37u!t?ui!H7a1zWoPY{lu7k ziWhtrgD*Wy7^#j?p+A9EBVgG}-aTAw>}48H(~!-n{a>$$|BU&qj`MZ;zrFrZ??QkGV002cxQELD)IV6CXA>vUzpqG!wo_1TOat>$+K88><0CpPeQ^Z1PF zQjbT%+1O6)7W#_Q;}8BrpffZbg+>#=;~#1B_0SsiFlFO#`7djOXtI_S-LhzQ3hp6{ zFeb;#iws1JkH%VU!y0f3r15QP=N8oq&s!<4d6zDxrL_Y}_y|`-lt`C#BefI^hk}O3 z@OB~o(AkBj>4Y4Bk|7t6qhbBx4jiIPVCN2iWMz-D+{OJ?F<9RQ6tzU{miQL91xe26 zz15LZ=Vt8c@lQGdu$9RX;*4i&rSTKaW@K|Y7@`ecDQDgh3YGqZQEkD6g3sEV_35m^ z3@m6W{MKS^0EL!4U#LsA$YRfk4hZlC8=zj*m11)ncS`0^pm78(t)xFWP{Bg3YUaQ%@k01ViVB7v< zn~3xYT~ZF3R?2mpcW~xJGf0Qm$82OEc8J0^12AS2 z-t%Fk1B7ry;#uc zG++s^8d9n|-Xm1RP7wls@KtKULL)K~d3*vSUOIfYC`*(v0b6NUxX0%MH}_%CG2igc zPUIk_i7y~yFdbFk+n$hM3_>)SeMvzTQpQUj2ZgK(Lcct&6r7-(`_n`9>D}P`HsWy- z($drA3P*YNSH%3R{DqOqPjsTivbiP|v_o~>5aW%yOGju?-$iFek55d$iLYsnYv@&d z+{>>5R-qR@PX%+!h0os~$gLc=gs+XOjT73ulNclo*>`zt>FM^+&n*}$K}~<<@BC_L l)xTxp|Jt)I_=R1{UnZ#uR~MbjJwC6!^XB+@O7}#c{|yPO2t)t? literal 0 HcmV?d00001 diff --git a/evidence/measurements.csv b/evidence/measurements.csv new file mode 100644 index 00000000..ae5f9105 --- /dev/null +++ b/evidence/measurements.csv @@ -0,0 +1,7 @@ +profile,repeat,completed,successful_rps,p50,p95,p99,error_fraction +320x200,1,80,3.982,0.401,0.7265,0.7436,0.0 +320x200,2,73,3.607,0.4219,0.7366,0.7857,0.0 +320x200,3,71,3.549,0.4389,0.7694,0.82,0.0 +1024x768,1,45,2.188,0.8308,1.1487,1.1615,0.0 +1024x768,2,32,1.569,0.79,1.1397,1.2381,0.030303030303030304 +1024x768,3,40,1.908,1.0675,1.2299,1.3224,0.0 diff --git a/evidence/normal-start-retry.txt b/evidence/normal-start-retry.txt new file mode 100644 index 00000000..b6177cfb --- /dev/null +++ b/evidence/normal-start-retry.txt @@ -0,0 +1,37 @@ + Container lyakhonov-lab5-redis-1 Running + Container lyakhonov-lab5-s3-1 Running + Container lyakhonov-lab5-db-1 Running + Container lyakhonov-lab5-broker-1 Running + Container lyakhonov-lab5-dispatcher-1 Running + Container lyakhonov-lab5-staging-cleaner-1 Running + Container lyakhonov-lab5-worker-1 Running + Container lyakhonov-lab5-worker-2 Running + Container lyakhonov-lab5-api-1 Running + Container lyakhonov-lab5-api-2 Running + Container lyakhonov-lab5-lb-1 Running + Container lyakhonov-lab5-bucket-init-1 Starting + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Started + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Starting + Container lyakhonov-lab5-migrate-1 Started + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-broker-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-broker-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-broker-1 Healthy + Container lyakhonov-lab5-broker-1 Healthy + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited diff --git a/evidence/normal-start.txt b/evidence/normal-start.txt new file mode 100644 index 00000000..4e3c8816 --- /dev/null +++ b/evidence/normal-start.txt @@ -0,0 +1,80 @@ + Container lyakhonov-lab5-redis-1 Running + Container lyakhonov-lab5-broker-1 Running + Container lyakhonov-lab5-db-1 Running + Container lyakhonov-lab5-s3-1 Running + Container lyakhonov-lab5-migrate-1 Recreate + Container lyakhonov-lab5-migrate-1 Recreated + Container lyakhonov-lab5-staging-cleaner-1 Recreate + Container lyakhonov-lab5-dispatcher-1 Recreate + Container lyakhonov-lab5-worker-2 Recreate + Container lyakhonov-lab5-api-2 Recreate + Container lyakhonov-lab5-api-1 Recreate + Container lyakhonov-lab5-worker-1 Recreate + Container lyakhonov-lab5-worker-2 Recreated + Container lyakhonov-lab5-worker-1 Recreated + Container lyakhonov-lab5-dispatcher-1 Recreated + Container lyakhonov-lab5-api-2 Recreated + Container lyakhonov-lab5-api-1 Recreated + Container lyakhonov-lab5-lb-1 Running + Container lyakhonov-lab5-staging-cleaner-1 Recreated + Container lyakhonov-lab5-bucket-init-1 Starting + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Started + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Starting + Container lyakhonov-lab5-migrate-1 Started + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-broker-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-broker-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-broker-1 Healthy + Container lyakhonov-lab5-broker-1 Healthy + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-staging-cleaner-1 Starting + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-worker-2 Starting + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-dispatcher-1 Starting + Container lyakhonov-lab5-api-1 Starting + Container lyakhonov-lab5-api-1 Started + Container lyakhonov-lab5-api-2 Starting + Container lyakhonov-lab5-staging-cleaner-1 Started + Container lyakhonov-lab5-dispatcher-1 Started + Container lyakhonov-lab5-worker-2 Started + Container lyakhonov-lab5-worker-1 Starting + Container lyakhonov-lab5-api-2 Started + Container lyakhonov-lab5-worker-1 Started + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-worker-2 Waiting + Container lyakhonov-lab5-worker-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-s3-1 Waiting + Container lyakhonov-lab5-broker-1 Waiting + Container lyakhonov-lab5-lb-1 Waiting + Container lyakhonov-lab5-dispatcher-1 Waiting + Container lyakhonov-lab5-staging-cleaner-1 Waiting + Container lyakhonov-lab5-api-1 Waiting + Container lyakhonov-lab5-api-2 Waiting + Container lyakhonov-lab5-redis-1 Waiting + Container lyakhonov-lab5-lb-1 Healthy + Container lyakhonov-lab5-redis-1 Healthy + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-s3-1 Healthy + Container lyakhonov-lab5-broker-1 Healthy + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-api-1 Healthy + Container lyakhonov-lab5-api-2 Healthy +container lyakhonov-lab5-worker-2 has no healthcheck configured diff --git a/evidence/persistence.json b/evidence/persistence.json new file mode 100644 index 00000000..0753eeca --- /dev/null +++ b/evidence/persistence.json @@ -0,0 +1,176 @@ +{ + "command": [ + "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-X", + "utf8", + "tests/resilience.py", + "--mode", + "persistence", + "--base-url", + "http://127.0.0.1:8085", + "--action-script", + "C:\\Users\\User\\Desktop\\vysoko-nagr 5\\scripts\\restart.cmd" + ], + "exit_code": 0, + "output": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.\n", + "before": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:56:56.938529239Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:56:57.198039404Z", + "running": true + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:56:31.315056049Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:58:21.664983091Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:56:56.914519489Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:58:22.174853427Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:56:56.92548889Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:56:57.356659637Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:56:56.927116397Z", + "running": true + } + ], + "after": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:58:31.589874697Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:58:31.785178375Z", + "running": true + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:58:21.664983091Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:58:22.174853427Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ] +} \ No newline at end of file diff --git a/evidence/pipeline-initial-run.txt b/evidence/pipeline-initial-run.txt new file mode 100644 index 00000000..9e7cbf46 --- /dev/null +++ b/evidence/pipeline-initial-run.txt @@ -0,0 +1,56 @@ +PASS: negative_upload_acl_and_24_concurrent_messages +PASS: accepted_survives_api_broker_restart +PASS: before_publish +PASS: after_publish +PASS: before_effect +PASS: after_original +PASS: after_variants +PASS: after_commit +PASS: before_publish +PASS: after_publish +PASS: before_effect +PASS: after_original +PASS: after_variants +PASS: after_commit +PASS: before_publish +PASS: after_publish +PASS: before_effect +PASS: after_original +PASS: after_variants +PASS: after_commit +Traceback (most recent call last): + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 234, in + Experiment(parser.parse_args()).main() + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 222, in main + self.versions() + ~~~~~~~~~~~~~^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 135, in versions + objects = self.probe('objects', id=att['id']) + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 35, in probe + check(result.returncode == 0, 'probe failed: ' + result.stderr) + ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\tests\client.py", line 24, in check + raise Failure(message) +client.Failure: probe failed: Traceback (most recent call last): + File "", line 54, in + File "", line 35, in run + File "/usr/local/lib/python3.11/site-packages/botocore/paginate.py", line 272, in __iter__ + response = self._make_request(current_kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/context.py", line 123, in wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/paginate.py", line 360, in _make_request + return self._method(**current_kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/client.py", line 602, in _api_call + return self._make_api_call(operation_name, kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/context.py", line 123, in wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/client.py", line 1078, in _make_api_call + raise error_class(parsed_response, operation_name) +botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the ListObjectsV2 operation: Access Denied. + diff --git a/evidence/pipeline-resumed-run.txt b/evidence/pipeline-resumed-run.txt new file mode 100644 index 00000000..a8fec748 --- /dev/null +++ b/evidence/pipeline-resumed-run.txt @@ -0,0 +1,8 @@ +PASS: stale_running_worker_and_reordered_duplicates +PASS: upgrade_ready_after_staging_cleanup +PASS: stale_running_worker_and_reordered_duplicates +PASS: upgrade_ready_after_staging_cleanup +PASS: stale_running_worker_and_reordered_duplicates +PASS: upgrade_ready_after_staging_cleanup +PASS: permanent_error_and_explicit_replay +PASS: transient_error_and_explicit_replay diff --git a/evidence/pipeline-run.txt b/evidence/pipeline-run.txt new file mode 100644 index 00000000..9e7cbf46 --- /dev/null +++ b/evidence/pipeline-run.txt @@ -0,0 +1,56 @@ +PASS: negative_upload_acl_and_24_concurrent_messages +PASS: accepted_survives_api_broker_restart +PASS: before_publish +PASS: after_publish +PASS: before_effect +PASS: after_original +PASS: after_variants +PASS: after_commit +PASS: before_publish +PASS: after_publish +PASS: before_effect +PASS: after_original +PASS: after_variants +PASS: after_commit +PASS: before_publish +PASS: after_publish +PASS: before_effect +PASS: after_original +PASS: after_variants +PASS: after_commit +Traceback (most recent call last): + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 234, in + Experiment(parser.parse_args()).main() + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 222, in main + self.versions() + ~~~~~~~~~~~~~^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 135, in versions + objects = self.probe('objects', id=att['id']) + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 35, in probe + check(result.returncode == 0, 'probe failed: ' + result.stderr) + ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\tests\client.py", line 24, in check + raise Failure(message) +client.Failure: probe failed: Traceback (most recent call last): + File "", line 54, in + File "", line 35, in run + File "/usr/local/lib/python3.11/site-packages/botocore/paginate.py", line 272, in __iter__ + response = self._make_request(current_kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/context.py", line 123, in wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/paginate.py", line 360, in _make_request + return self._method(**current_kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/client.py", line 602, in _api_call + return self._make_api_call(operation_name, kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/context.py", line 123, in wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/client.py", line 1078, in _make_api_call + raise error_class(parsed_response, operation_name) +botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the ListObjectsV2 operation: Access Denied. + diff --git a/evidence/pipeline.json b/evidence/pipeline.json new file mode 100644 index 00000000..37fcdcdb --- /dev/null +++ b/evidence/pipeline.json @@ -0,0 +1,5145 @@ +[ + { + "scenario": "negative_upload_acl_and_24_concurrent_messages", + "seconds": 4.168, + "checks": 9, + "message_ids": 1 + }, + { + "scenario": "accepted_survives_api_broker_restart", + "seconds": 27.593, + "attachment_id": "619fa63f-4fc4-4022-9ebb-d45ac01ac375", + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:41:58.297145512Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:41:58.683174598Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:39:52.157659158Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:41:55.56150084Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:41:58.280887966Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:41:56.049697978Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:41:58.292710818Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:41:58.616417234Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:42:17.221143902Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:41:58.280887966Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:42:17.723111805Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:42:33.06126539Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:42:33.37189925Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "before_publish", + "seconds": 32.205, + "repeat": 1, + "attachment_id": "de278c50-0dea-4db7-bbc5-633e03ae90ad", + "fault": { + "job_id": "5112c4f0-fee6-4814-bf47-09f0408f5b7d", + "point": "before_publish", + "action": "crash", + "hit_at": "2026-11-20 03:42:50.467465+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "5112c4f0-fee6-4814-bf47-09f0408f5b7d", + "attachment_id": "de278c50-0dea-4db7-bbc5-633e03ae90ad", + "staging_key": "staging/de278c50-0dea-4db7-bbc5-633e03ae90ad", + "created_at": "2026-11-20 03:42:39.379658+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:43:05.529083+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:43:05.762115+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:42:17.221143902Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:41:58.280887966Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:42:17.723111805Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:42:33.06126539Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:42:33.37189925Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:42:44.051926228Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:42:50.690235738Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:42:44.545702483Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:42:33.06126539Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:42:33.37189925Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_publish", + "seconds": 16.65, + "repeat": 1, + "attachment_id": "132c66de-7282-45a3-a200-11d33b54e2ed", + "fault": { + "job_id": "df29a891-03c2-49e2-990c-1cd835743ea0", + "point": "after_publish", + "action": "crash", + "hit_at": "2026-11-20 03:43:20.963939+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "df29a891-03c2-49e2-990c-1cd835743ea0", + "attachment_id": "132c66de-7282-45a3-a200-11d33b54e2ed", + "staging_key": "staging/132c66de-7282-45a3-a200-11d33b54e2ed", + "created_at": "2026-11-20 03:43:11.552216+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": null, + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:43:21.503078+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:42:44.051926228Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:42:50.690235738Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:42:44.545702483Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:42:33.06126539Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:42:33.37189925Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:43:16.580731829Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:43:17.066108428Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:42:33.06126539Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:42:33.37189925Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "before_effect", + "seconds": 28.623, + "repeat": 1, + "attachment_id": "ec2c7c0c-6a39-4715-944e-8faf09b2279f", + "fault": { + "job_id": "06dbadb4-c510-4c08-b28d-5a0dd00a5b02", + "point": "before_effect", + "action": "crash", + "hit_at": "2026-11-20 03:43:37.411076+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "06dbadb4-c510-4c08-b28d-5a0dd00a5b02", + "attachment_id": "ec2c7c0c-6a39-4715-944e-8faf09b2279f", + "staging_key": "staging/ec2c7c0c-6a39-4715-944e-8faf09b2279f", + "created_at": "2026-11-20 03:43:28.547524+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:43:49.044160+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:43:49.465850+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:43:16.580731829Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:43:17.066108428Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:42:33.06126539Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:42:33.37189925Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:43:33.084987014Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:43:33.552597501Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:43:37.634736588Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:43:36.683144593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_original", + "seconds": 26.509, + "repeat": 1, + "attachment_id": "62f6f8a5-a3c2-4a5b-ad26-d035b9511998", + "fault": { + "job_id": "55a16d30-7c35-454c-86fd-d689757a51e6", + "point": "after_original", + "action": "crash", + "hit_at": "2026-11-20 03:44:05.931398+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "55a16d30-7c35-454c-86fd-d689757a51e6", + "attachment_id": "62f6f8a5-a3c2-4a5b-ad26-d035b9511998", + "staging_key": "staging/62f6f8a5-a3c2-4a5b-ad26-d035b9511998", + "created_at": "2026-11-20 03:43:57.216845+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:44:17.192528+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:44:17.509618+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:43:33.084987014Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:43:33.552597501Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:43:37.634736588Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:43:36.683144593Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:44:01.486596866Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:44:01.983245268Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:44:06.210667354Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:04.953115416Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_variants", + "seconds": 27.131, + "repeat": 1, + "attachment_id": "78bb9cfa-55c4-46d0-bc39-548f8311fbd0", + "fault": { + "job_id": "3a5263bf-c4e3-4516-abc7-7e5e2b8da5e4", + "point": "after_variants", + "action": "crash", + "hit_at": "2026-11-20 03:44:32.808598+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "3a5263bf-c4e3-4516-abc7-7e5e2b8da5e4", + "attachment_id": "78bb9cfa-55c4-46d0-bc39-548f8311fbd0", + "staging_key": "staging/78bb9cfa-55c4-46d0-bc39-548f8311fbd0", + "created_at": "2026-11-20 03:44:23.725433+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:44:43.705309+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:44:43.969458+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:44:01.486596866Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:44:01.983245268Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:44:06.210667354Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:04.953115416Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:44:27.912594847Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:44:28.369270089Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:44:33.022103249Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:31.933900075Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_commit", + "seconds": 17.081, + "repeat": 1, + "attachment_id": "352b23af-8c50-4927-bf13-70a1bb6fce21", + "fault": { + "job_id": "ecf012e7-06b4-4ae4-a45d-70028bfc12e4", + "point": "after_commit", + "action": "crash", + "hit_at": "2026-11-20 03:44:59.812097+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "ecf012e7-06b4-4ae4-a45d-70028bfc12e4", + "attachment_id": "352b23af-8c50-4927-bf13-70a1bb6fce21", + "staging_key": "staging/352b23af-8c50-4927-bf13-70a1bb6fce21", + "created_at": "2026-11-20 03:44:50.799790+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:44:50.989770+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:44:59.810079+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:44:27.912594847Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:44:28.369270089Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:44:33.022103249Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:31.933900075Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:44:55.193771438Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:44:55.695401835Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:00.069022774Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:58.720463171Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "before_publish", + "seconds": 28.94, + "repeat": 2, + "attachment_id": "dd45ec28-ca62-4702-83cc-0ad183fa73f0", + "fault": { + "job_id": "055f56e9-86fd-42c9-8b20-5dc72011918b", + "point": "before_publish", + "action": "crash", + "hit_at": "2026-11-20 03:45:15.188522+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "055f56e9-86fd-42c9-8b20-5dc72011918b", + "attachment_id": "dd45ec28-ca62-4702-83cc-0ad183fa73f0", + "staging_key": "staging/dd45ec28-ca62-4702-83cc-0ad183fa73f0", + "created_at": "2026-11-20 03:45:07.608313+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:45:30.320391+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:45:30.632786+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:44:55.193771438Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:44:55.695401835Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:00.069022774Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:58.720463171Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:45:11.239538513Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:15.431620016Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:45:11.825599429Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:00.069022774Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:58.720463171Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_publish", + "seconds": 15.182, + "repeat": 2, + "attachment_id": "eaf40096-5053-4ef3-a9db-14dfc234a8ce", + "fault": { + "job_id": "d4c4afd1-7e37-447d-8c41-2ebad3bca874", + "point": "after_publish", + "action": "crash", + "hit_at": "2026-11-20 03:45:43.998330+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "d4c4afd1-7e37-447d-8c41-2ebad3bca874", + "attachment_id": "eaf40096-5053-4ef3-a9db-14dfc234a8ce", + "staging_key": "staging/eaf40096-5053-4ef3-a9db-14dfc234a8ce", + "created_at": "2026-11-20 03:45:36.404146+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": null, + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:45:44.388768+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:45:11.239538513Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:15.431620016Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:45:11.825599429Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:00.069022774Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:58.720463171Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:45:40.152387954Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:45:40.655870854Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:00.069022774Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:58.720463171Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "before_effect", + "seconds": 25.678, + "repeat": 2, + "attachment_id": "1949b244-780e-4a3e-8be2-918310eb0854", + "fault": { + "job_id": "eb32b287-eed7-4336-8435-a170219d0751", + "point": "before_effect", + "action": "crash", + "hit_at": "2026-11-20 03:45:59.502654+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "eb32b287-eed7-4336-8435-a170219d0751", + "attachment_id": "1949b244-780e-4a3e-8be2-918310eb0854", + "staging_key": "staging/1949b244-780e-4a3e-8be2-918310eb0854", + "created_at": "2026-11-20 03:45:51.858645+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:46:11.045887+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:46:11.339887+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:45:40.152387954Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:45:40.655870854Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:00.069022774Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:58.720463171Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:45:56.13119465Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:45:56.615857607Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:59.755318954Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:45:58.61301125Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_original", + "seconds": 25.642, + "repeat": 2, + "attachment_id": "85b5126d-f0e3-499d-aca8-6c13e1818e28", + "fault": { + "job_id": "bd85defc-0084-4d19-b20f-2421363745b1", + "point": "after_original", + "action": "crash", + "hit_at": "2026-11-20 03:46:25.817634+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "bd85defc-0084-4d19-b20f-2421363745b1", + "attachment_id": "85b5126d-f0e3-499d-aca8-6c13e1818e28", + "staging_key": "staging/85b5126d-f0e3-499d-aca8-6c13e1818e28", + "created_at": "2026-11-20 03:46:17.558732+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:46:36.887135+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:46:37.193120+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:45:56.13119465Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:45:56.615857607Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:59.755318954Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:45:58.61301125Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:46:21.542059999Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:46:22.037531019Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:46:26.027818995Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:46:24.617098765Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_variants", + "seconds": 25.458, + "repeat": 2, + "attachment_id": "b84b0b1f-3265-494d-a179-9f71daa68613", + "fault": { + "job_id": "ac944804-f74f-4ce5-9d25-22c4565b7887", + "point": "after_variants", + "action": "crash", + "hit_at": "2026-11-20 03:46:51.060205+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "ac944804-f74f-4ce5-9d25-22c4565b7887", + "attachment_id": "b84b0b1f-3265-494d-a179-9f71daa68613", + "staging_key": "staging/b84b0b1f-3265-494d-a179-9f71daa68613", + "created_at": "2026-11-20 03:46:43.356889+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:47:02.105387+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:47:02.403063+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:46:21.542059999Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:46:22.037531019Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:46:26.027818995Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:46:24.617098765Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:46:47.067326151Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:46:47.551997225Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:46:51.298628199Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:46:49.994100608Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_commit", + "seconds": 14.989, + "repeat": 2, + "attachment_id": "8e07fb74-6f9d-4cb1-a79e-1b791fc70534", + "fault": { + "job_id": "1f4c65ab-6330-4b9e-b2f0-d661ba79b963", + "point": "after_commit", + "action": "crash", + "hit_at": "2026-11-20 03:47:16.519594+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "1f4c65ab-6330-4b9e-b2f0-d661ba79b963", + "attachment_id": "8e07fb74-6f9d-4cb1-a79e-1b791fc70534", + "staging_key": "staging/8e07fb74-6f9d-4cb1-a79e-1b791fc70534", + "created_at": "2026-11-20 03:47:08.839253+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:47:09.075864+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:47:16.517978+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:46:47.067326151Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:46:47.551997225Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:46:51.298628199Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:46:49.994100608Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:47:12.483349362Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:47:12.913604465Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:47:16.742277702Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:47:15.490450932Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "before_publish", + "seconds": 29.804, + "repeat": 3, + "attachment_id": "d75d929a-6d7f-466b-b26f-7e72c2407a5a", + "fault": { + "job_id": "dd9ba48e-f4b6-439e-b6f1-82e3400d3538", + "point": "before_publish", + "action": "crash", + "hit_at": "2026-11-20 03:47:31.744254+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "dd9ba48e-f4b6-439e-b6f1-82e3400d3538", + "attachment_id": "d75d929a-6d7f-466b-b26f-7e72c2407a5a", + "staging_key": "staging/d75d929a-6d7f-466b-b26f-7e72c2407a5a", + "created_at": "2026-11-20 03:47:23.446790+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:47:46.792588+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:47:47.081130+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:47:12.483349362Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:47:12.913604465Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:47:16.742277702Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:47:15.490450932Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:47:27.559641991Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:47:31.929204122Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:47:28.057242867Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:47:16.742277702Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:47:15.490450932Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_publish", + "seconds": 14.478, + "repeat": 3, + "attachment_id": "0306a14e-ff85-4e84-8455-2802974bd23e", + "fault": { + "job_id": "5a36ea65-1082-4564-8bd9-aeb4bebc3cf3", + "point": "after_publish", + "action": "crash", + "hit_at": "2026-11-20 03:48:01.004583+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "5a36ea65-1082-4564-8bd9-aeb4bebc3cf3", + "attachment_id": "0306a14e-ff85-4e84-8455-2802974bd23e", + "staging_key": "staging/0306a14e-ff85-4e84-8455-2802974bd23e", + "created_at": "2026-11-20 03:47:53.380497+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": null, + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:48:01.416005+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:47:27.559641991Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:47:31.929204122Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:47:28.057242867Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:47:16.742277702Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:47:15.490450932Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:47:57.217828378Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:47:57.690157191Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:47:16.742277702Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:47:15.490450932Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "before_effect", + "seconds": 26.598, + "repeat": 3, + "attachment_id": "606f90f6-9e67-4082-b961-5b2bca12ec2a", + "fault": { + "job_id": "d90be4cb-5786-4553-b770-c25e59fdcd18", + "point": "before_effect", + "action": "crash", + "hit_at": "2026-11-20 03:48:16.197409+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "d90be4cb-5786-4553-b770-c25e59fdcd18", + "attachment_id": "606f90f6-9e67-4082-b961-5b2bca12ec2a", + "staging_key": "staging/606f90f6-9e67-4082-b961-5b2bca12ec2a", + "created_at": "2026-11-20 03:48:08.004079+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:48:28.023266+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:48:28.383964+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:47:57.217828378Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:47:57.690157191Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:47:16.742277702Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:47:15.490450932Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:48:12.202153026Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:48:12.697238174Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:48:16.424435299Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:48:15.687396384Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_original", + "seconds": 26.292, + "repeat": 3, + "attachment_id": "917d592f-eb5f-4c07-8a2b-d5020414c012", + "fault": { + "job_id": "0d555a0c-4c0e-48bc-8b5e-001b0e78a2c8", + "point": "after_original", + "action": "crash", + "hit_at": "2026-11-20 03:48:42.768054+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "0d555a0c-4c0e-48bc-8b5e-001b0e78a2c8", + "attachment_id": "917d592f-eb5f-4c07-8a2b-d5020414c012", + "staging_key": "staging/917d592f-eb5f-4c07-8a2b-d5020414c012", + "created_at": "2026-11-20 03:48:34.502820+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:48:54.167617+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:48:54.418755+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:48:12.202153026Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:48:12.697238174Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:48:16.424435299Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:48:15.687396384Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:48:38.330040227Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:48:38.829333153Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:48:43.018521035Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:48:41.772811802Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_variants", + "seconds": 25.466, + "repeat": 3, + "attachment_id": "5229c691-7f61-445e-8922-47c8dcb92b99", + "fault": { + "job_id": "f46470fb-d143-4c09-a40c-b05674b33e69", + "point": "after_variants", + "action": "crash", + "hit_at": "2026-11-20 03:49:08.669526+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "f46470fb-d143-4c09-a40c-b05674b33e69", + "attachment_id": "5229c691-7f61-445e-8922-47c8dcb92b99", + "staging_key": "staging/5229c691-7f61-445e-8922-47c8dcb92b99", + "created_at": "2026-11-20 03:49:00.638699+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:49:19.711271+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:49:20.069410+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:48:38.330040227Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:48:38.829333153Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:48:43.018521035Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:48:41.772811802Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:49:04.391235089Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:49:04.865540417Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:49:08.946091975Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:49:07.317159304Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_commit", + "seconds": 16.739, + "repeat": 3, + "attachment_id": "32278609-0542-4c9e-9ddf-7238d78966f7", + "fault": { + "job_id": "5b89ba7a-caa6-4364-ad27-befa2904c313", + "point": "after_commit", + "action": "crash", + "hit_at": "2026-11-20 03:49:35.327700+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "5b89ba7a-caa6-4364-ad27-befa2904c313", + "attachment_id": "32278609-0542-4c9e-9ddf-7238d78966f7", + "staging_key": "staging/32278609-0542-4c9e-9ddf-7238d78966f7", + "created_at": "2026-11-20 03:49:26.526430+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:49:26.589812+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:49:35.325775+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:49:04.391235089Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:49:04.865540417Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:49:08.946091975Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:49:07.317159304Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:49:30.614157138Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:49:31.125905282Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:49:35.57674716Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:49:34.180088959Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "stale_running_worker_and_reordered_duplicates", + "seconds": 45.964, + "repeat": 1, + "attachment_id": "bd690fb9-1393-4fd8-94fe-0bcc4c78f632", + "before_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + }, + "after_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + }, + "versions": [ + { + "pipeline_version": 1, + "state": "superseded", + "attempt": 1 + }, + { + "pipeline_version": 2, + "state": "ready", + "attempt": 1 + } + ], + "objects": [ + { + "key": "originals/bd690fb9-1393-4fd8-94fe-0bcc4c78f632", + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "size": 111817 + }, + { + "key": "variants/bd690fb9-1393-4fd8-94fe-0bcc4c78f632/v1/crop", + "etag": "\"989d73cfe100e73f9b13d8260e1c2e30\"", + "size": 2462 + }, + { + "key": "variants/bd690fb9-1393-4fd8-94fe-0bcc4c78f632/v1/optimized", + "etag": "\"14bfc73d33bbab974088b27794a8e34a\"", + "size": 2576 + }, + { + "key": "variants/bd690fb9-1393-4fd8-94fe-0bcc4c78f632/v1/thumbnail", + "etag": "\"e31836875f24868250c5fc4469d1c8d0\"", + "size": 5813 + }, + { + "key": "variants/bd690fb9-1393-4fd8-94fe-0bcc4c78f632/v2/crop", + "etag": "\"f377b4ca5f884179377dc3189b8e10af\"", + "size": 2888 + }, + { + "key": "variants/bd690fb9-1393-4fd8-94fe-0bcc4c78f632/v2/optimized", + "etag": "\"4e41ece95aee8e9fdf8097022e10da9e\"", + "size": 3152 + }, + { + "key": "variants/bd690fb9-1393-4fd8-94fe-0bcc4c78f632/v2/thumbnail", + "etag": "\"85e637fcc90201e8a3a98996b4025373\"", + "size": 7014 + } + ] + }, + { + "scenario": "upgrade_ready_after_staging_cleanup", + "seconds": 11.151, + "repeat": 1, + "attachment_id": "77489ca8-15ee-4b8d-a21f-d310454650e6", + "before_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "crop": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "optimized": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c" + }, + "after_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + } + }, + { + "scenario": "stale_running_worker_and_reordered_duplicates", + "seconds": 45.456, + "repeat": 2, + "attachment_id": "3ce6bb5f-9194-451c-b27e-02a6ec63881a", + "before_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + }, + "after_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + }, + "versions": [ + { + "pipeline_version": 1, + "state": "superseded", + "attempt": 1 + }, + { + "pipeline_version": 2, + "state": "ready", + "attempt": 1 + } + ], + "objects": [ + { + "key": "originals/3ce6bb5f-9194-451c-b27e-02a6ec63881a", + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "size": 111817 + }, + { + "key": "variants/3ce6bb5f-9194-451c-b27e-02a6ec63881a/v1/crop", + "etag": "\"989d73cfe100e73f9b13d8260e1c2e30\"", + "size": 2462 + }, + { + "key": "variants/3ce6bb5f-9194-451c-b27e-02a6ec63881a/v1/optimized", + "etag": "\"14bfc73d33bbab974088b27794a8e34a\"", + "size": 2576 + }, + { + "key": "variants/3ce6bb5f-9194-451c-b27e-02a6ec63881a/v1/thumbnail", + "etag": "\"e31836875f24868250c5fc4469d1c8d0\"", + "size": 5813 + }, + { + "key": "variants/3ce6bb5f-9194-451c-b27e-02a6ec63881a/v2/crop", + "etag": "\"f377b4ca5f884179377dc3189b8e10af\"", + "size": 2888 + }, + { + "key": "variants/3ce6bb5f-9194-451c-b27e-02a6ec63881a/v2/optimized", + "etag": "\"4e41ece95aee8e9fdf8097022e10da9e\"", + "size": 3152 + }, + { + "key": "variants/3ce6bb5f-9194-451c-b27e-02a6ec63881a/v2/thumbnail", + "etag": "\"85e637fcc90201e8a3a98996b4025373\"", + "size": 7014 + } + ] + }, + { + "scenario": "upgrade_ready_after_staging_cleanup", + "seconds": 11.485, + "repeat": 2, + "attachment_id": "563c45e8-537d-4634-a1c3-0037468836f9", + "before_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "crop": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "optimized": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c" + }, + "after_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + } + }, + { + "scenario": "stale_running_worker_and_reordered_duplicates", + "seconds": 64.377, + "repeat": 3, + "attachment_id": "7117c2f5-48d1-4227-afa0-a2ee439a067e", + "before_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + }, + "after_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + }, + "versions": [ + { + "pipeline_version": 1, + "state": "superseded", + "attempt": 1 + }, + { + "pipeline_version": 2, + "state": "ready", + "attempt": 1 + } + ], + "objects": [ + { + "key": "originals/7117c2f5-48d1-4227-afa0-a2ee439a067e", + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "size": 111817 + }, + { + "key": "variants/7117c2f5-48d1-4227-afa0-a2ee439a067e/v1/crop", + "etag": "\"989d73cfe100e73f9b13d8260e1c2e30\"", + "size": 2462 + }, + { + "key": "variants/7117c2f5-48d1-4227-afa0-a2ee439a067e/v1/optimized", + "etag": "\"14bfc73d33bbab974088b27794a8e34a\"", + "size": 2576 + }, + { + "key": "variants/7117c2f5-48d1-4227-afa0-a2ee439a067e/v1/thumbnail", + "etag": "\"e31836875f24868250c5fc4469d1c8d0\"", + "size": 5813 + }, + { + "key": "variants/7117c2f5-48d1-4227-afa0-a2ee439a067e/v2/crop", + "etag": "\"f377b4ca5f884179377dc3189b8e10af\"", + "size": 2888 + }, + { + "key": "variants/7117c2f5-48d1-4227-afa0-a2ee439a067e/v2/optimized", + "etag": "\"4e41ece95aee8e9fdf8097022e10da9e\"", + "size": 3152 + }, + { + "key": "variants/7117c2f5-48d1-4227-afa0-a2ee439a067e/v2/thumbnail", + "etag": "\"85e637fcc90201e8a3a98996b4025373\"", + "size": 7014 + } + ] + }, + { + "scenario": "upgrade_ready_after_staging_cleanup", + "seconds": 15.628, + "repeat": 3, + "attachment_id": "04003138-9d81-4a15-b6d9-c16a12a79228", + "before_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "crop": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "optimized": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c" + }, + "after_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + } + }, + { + "scenario": "permanent_error_and_explicit_replay", + "seconds": 22.401, + "attachment_id": "77a4b759-5a49-4633-8152-1038fe144ce7", + "attempts": [ + { + "attempt": 1, + "state": "failed", + "next_publish_at": null, + "last_error": "staging_integrity" + } + ], + "audit": [ + { + "attempt": 2, + "reason": "operator" + } + ] + }, + { + "scenario": "transient_error_and_explicit_replay", + "seconds": 44.885, + "attachment_id": "6601079a-655c-4827-9174-b2b1c57685d2", + "attempts": [ + { + "attempt": 2, + "state": "queued", + "next_publish_at": "2026-11-20 03:55:16.096457+00:00", + "last_error": "processing_failed" + }, + { + "attempt": 4, + "state": "queued", + "next_publish_at": "2026-11-20 03:55:22.676112+00:00", + "last_error": "processing_failed" + }, + { + "attempt": 5, + "state": "queued", + "next_publish_at": "2026-11-20 03:55:30.492215+00:00", + "last_error": "processing_failed" + }, + { + "attempt": 5, + "state": "queued", + "next_publish_at": "2026-11-20 03:55:30.492215+00:00", + "last_error": "processing_failed" + }, + { + "attempt": 5, + "state": "queued", + "next_publish_at": "2026-11-20 03:55:30.492215+00:00", + "last_error": "processing_failed" + }, + { + "attempt": 5, + "state": "failed", + "next_publish_at": null, + "last_error": "processing_failed" + } + ], + "audit": [ + { + "attempt": 6, + "reason": "operator" + } + ] + }, + { + "scenario": "negative_upload_acl_and_24_concurrent_messages", + "seconds": 12.022, + "checks": 9, + "message_ids": 1 + }, + { + "scenario": "accepted_survives_api_broker_restart", + "seconds": 35.415, + "attachment_id": "1450e655-aebb-4b55-8196-7823df0fb275", + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:55:53.440888182Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:55:54.012916611Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:55:13.450036277Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:55:13.735876724Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:56:30.629418613Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:56:30.739564284Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:56:31.315056049Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:56:32.571405318Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:56:33.084907991Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:56:40.43708125Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:56:40.643246654Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + } +] \ No newline at end of file diff --git a/evidence/resilience-initial-run.txt b/evidence/resilience-initial-run.txt new file mode 100644 index 00000000..27595d61 --- /dev/null +++ b/evidence/resilience-initial-run.txt @@ -0,0 +1,37 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Starting + Container lyakhonov-lab5-bucket-init-1 Started + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Starting + Container lyakhonov-lab5-migrate-1 Started + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 88, "failed_attempts": 4, "first_recovery_seconds": 2.856} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + + Container lyakhonov-lab5-bucket-init-1 Starting + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Started + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Starting + Container lyakhonov-lab5-migrate-1 Started + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-api-2 Starting + Container lyakhonov-lab5-api-2 Started +Traceback (most recent call last): + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\resilience_driver.py", line 82, in + targeted() + ~~~~~~~~^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\resilience_driver.py", line 62, in targeted + messages = list(page['items']) + ~~~~^^^^^^^^^ +TypeError: 'NoneType' object is not subscriptable diff --git a/evidence/resilience-run.txt b/evidence/resilience-run.txt new file mode 100644 index 00000000..27595d61 --- /dev/null +++ b/evidence/resilience-run.txt @@ -0,0 +1,37 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Starting + Container lyakhonov-lab5-bucket-init-1 Started + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Starting + Container lyakhonov-lab5-migrate-1 Started + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 88, "failed_attempts": 4, "first_recovery_seconds": 2.856} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + + Container lyakhonov-lab5-bucket-init-1 Starting + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Started + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Starting + Container lyakhonov-lab5-migrate-1 Started + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-api-2 Starting + Container lyakhonov-lab5-api-2 Started +Traceback (most recent call last): + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\resilience_driver.py", line 82, in + targeted() + ~~~~~~~~^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\resilience_driver.py", line 62, in targeted + messages = list(page['items']) + ~~~~^^^^^^^^^ +TypeError: 'NoneType' object is not subscriptable diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..5c90482b --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/structure.txt b/evidence/structure.txt new file mode 100644 index 00000000..b62af9c5 --- /dev/null +++ b/evidence/structure.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/unit.txt b/evidence/unit.txt new file mode 100644 index 00000000..3e7cac78 --- /dev/null +++ b/evidence/unit.txt @@ -0,0 +1,5 @@ +.. +---------------------------------------------------------------------- +Ran 2 tests in 0.142s + +OK diff --git a/evidence/validation.txt b/evidence/validation.txt new file mode 100644 index 00000000..248343ee --- /dev/null +++ b/evidence/validation.txt @@ -0,0 +1 @@ +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/requirements-research.txt b/requirements-research.txt new file mode 100644 index 00000000..d8323040 --- /dev/null +++ b/requirements-research.txt @@ -0,0 +1,2 @@ +-r requirements-dev.txt +matplotlib==3.10.7 diff --git a/scripts/audit.py b/scripts/audit.py index 11e5a68a..cfc4fc8c 100644 --- a/scripts/audit.py +++ b/scripts/audit.py @@ -41,7 +41,7 @@ def main(): assert not path.startswith('secrets/') and path != '.env', 'Tracked private state' content = (ROOT / path).read_bytes() assert not any(value and value in content for value in secret_values), 'Local secret leaked in ' + path - assert b'-----BEGIN PRIVATE KEY-----' not in content, 'Private key found' + assert (b'-----BEGIN ' + b'PRIVATE KEY-----') not in content, 'Private key found in ' + path authors = git(ROOT, 'log', '276dbda..HEAD', '--format=%h|%an|%ae|%cn|%ce').splitlines() for row in authors: assert row.split('|')[1:] == ['Sergey Lyakhonov', 'lyserg119@gmail.com'] * 2, 'Wrong author/committer' diff --git a/scripts/environment.py b/scripts/environment.py index 12c4afd6..f45db923 100644 --- a/scripts/environment.py +++ b/scripts/environment.py @@ -14,6 +14,7 @@ def output(*args): ids = output('docker', 'compose', '-p', 'lyakhonov-lab5', 'ps', '-q').split() containers = json.loads(output('docker', 'inspect', *ids)) result = dict(commit=output('git', 'rev-parse', 'HEAD'), python=platform.python_version(), + working_tree_dirty=bool(output('git', 'status', '--porcelain')), docker=json.loads(output('docker', 'version', '--format', '{{json .}}')), compose=output('docker', 'compose', 'version'), engine=json.loads(output('docker', 'info', '--format', '{"memory_bytes":{{.MemTotal}},"cpus":{{.NCPU}}}')), @@ -21,6 +22,7 @@ result = dict(commit=output('git', 'rev-parse', 'HEAD'), python=platform.python_ for row in containers: result['services'].append(dict(service=row['Config']['Labels']['com.docker.compose.service'], id=row['Id'], image=row['Config']['Image'], image_id=row['Image'], started_at=row['State']['StartedAt'], + restart_count=row['RestartCount'], oom_killed=row['State'].get('OOMKilled', False), memory_limit=row['HostConfig']['Memory'], nanocpus=row['HostConfig']['NanoCpus'], ports=row['HostConfig']['PortBindings'], settings=[x for x in row['Config']['Env'] if x.startswith(('JOB_', 'WORKER_', 'LAB5_FAULT_', 'STAGING_RETENTION_'))])) diff --git a/scripts/final_state.py b/scripts/final_state.py new file mode 100644 index 00000000..5555d27b --- /dev/null +++ b/scripts/final_state.py @@ -0,0 +1,36 @@ +"""Финальные агрегаты собственного стенда без пользовательских данных и секретов.""" +import json +from pathlib import Path +import subprocess +import re + +ROOT = Path(__file__).resolve().parents[1] +load = json.loads((ROOT / 'evidence/load-pipeline.json').read_text(encoding='utf-8')) +failed_ids = sorted(set(re.findall(r'[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', + ' '.join(error for row in load for error in row['failures'])))) +script = 'FAILED_IDS=' + repr(failed_ids) + '\n' + ''' +import json +from src.storage import storage +storage.pool.open(wait=True) +try: + with storage.pool.connection() as conn: + result = { + 'attachments': conn.execute('SELECT state,count(*) AS count FROM attachments GROUP BY state').fetchall(), + 'jobs': conn.execute('SELECT state,pipeline_version,count(*) AS count FROM attachment_jobs GROUP BY state,pipeline_version ORDER BY pipeline_version,state').fetchall(), + 'replays': conn.execute('SELECT count(*) AS count FROM attachment_replays').fetchone()['count'], + 'load_observation_errors': conn.execute('SELECT id,state,published_pipeline_version,target_pipeline_version FROM attachments WHERE id=ANY(%s::uuid[])', (FAILED_IDS,)).fetchall(), + 'violations': conn.execute("""SELECT count(*) AS count FROM attachments a + WHERE state <> 'ready' OR published_pipeline_version <> target_pipeline_version + OR (SELECT count(*) FROM attachment_variants v WHERE v.attachment_id=a.id + AND v.pipeline_version=a.published_pipeline_version) <> 3""").fetchone()['count'] + } + print(json.dumps(result, default=str)) +finally: + storage.pool.close() +''' +output = subprocess.check_output(['docker','compose','-p','lyakhonov-lab5','exec','-T','--index','1','api','python','-'], + input=script, cwd=ROOT, text=True, encoding='utf-8') +result = json.loads(output) +(ROOT / 'evidence/final-state.json').write_text(json.dumps(result, indent=2), encoding='utf-8') +assert result['violations'] == 0, 'Incomplete or inconsistent current manifests' +print(output) diff --git a/scripts/reproduce.py b/scripts/reproduce.py index cc5baaef..bcd37a79 100644 --- a/scripts/reproduce.py +++ b/scripts/reproduce.py @@ -30,27 +30,30 @@ def main(): (directory / 'repro-override.json').write_text(json.dumps(override), encoding='utf-8') compose = ['docker', 'compose', '-p', name, '-f', 'compose.yaml', '-f', 'repro-override.json'] events = [] - def run(args): + def run(args, required=True): result = subprocess.run(args, cwd=directory, env=env, text=True, encoding='utf-8', capture_output=True) # Только безопасные последние строки: ни inspect .Config.Env, ни secret files. events.append(dict(command=args, exit_code=result.returncode, output=(result.stdout + result.stderr)[-5000:])) - if result.returncode: + if result.returncode and required: raise RuntimeError('reproduction failed: ' + ' '.join(args)) try: run([sys.executable, '-X', 'utf8', 'scripts/bootstrap.py']) run([sys.executable, '-X', 'utf8', 'scripts/check_contract.py']) run(['docker', 'build', '-t', images[0], '.']) run(['docker', 'build', '-t', images[1], '-f', 'Dockerfile.s3', '.']) - run([*compose, 'up', '-d', '--wait']) + run([*compose, 'up', '-d']) + run([sys.executable, '-X', 'utf8', 'scripts/wait_ready.py', '--project', name, '--base-url', f'http://127.0.0.1:{port}']) run([sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', f'http://127.0.0.1:{port}']) run([*compose, 'ps']) finally: - run([*compose, 'down', '--volumes', '--remove-orphans']) + run([*compose, 'down', '--volumes', '--remove-orphans'], required=False) for tag in images: - run(['docker', 'image', 'rm', tag]) + run(['docker', 'image', 'rm', tag], required=False) result = dict(project=name, directory=str(directory), port=port, commit=subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), events=events) (ROOT / 'evidence/reproduction.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8') + if any(event['exit_code'] for event in events): + raise SystemExit('Reproduction or cleanup failed; see evidence/reproduction.json') print('PASS: clean export, fresh secrets, isolated project; temporary Docker resources removed') diff --git a/scripts/summarize.py b/scripts/summarize.py new file mode 100644 index 00000000..a9c9e055 --- /dev/null +++ b/scripts/summarize.py @@ -0,0 +1,43 @@ +"""Таблица отчёта из JSON; без ручного переноса значений.""" +import csv +import json +from pathlib import Path +import re + +ROOT = Path(__file__).resolve().parents[1] +load = json.loads((ROOT / 'evidence/load-pipeline.json').read_text(encoding='utf-8')) +faults = json.loads((ROOT / 'evidence/pipeline.json').read_text(encoding='utf-8')) +text = ['| Профиль | Bytes/image | Завершено за 3 повтора | Успешных image/s (min–max) | p50 ready, s | p95 ready, s | p99 ready, s | Ошибки |', + '| --- | --- | --- | --- | --- | --- | --- | --- |'] +for profile in sorted({r['profile'] for r in load}): + rows = [r for r in load if r['profile'] == profile] + def span(key): + return f"{min(r[key] for r in rows):.3f}–{max(r[key] for r in rows):.3f}" + text.append(f"| {profile} | {rows[0]['bytes_per_image']} | {sum(r['completed'] for r in rows)} | {span('successful_rps')} | {span('p50')} | {span('p95')} | {span('p99')} | {sum(len(r['failures']) for r in rows)} |") +def mib(value): + number, unit = re.match(r'([0-9.]+)([A-Za-z]+)', value.strip()).groups() + return float(number) * {'B':1/1048576,'KiB':1/1024,'MiB':1,'GiB':1024,'kB':1000/1048576,'MB':1000000/1048576,'GB':1000000000/1048576}[unit] +snapshots = [s for r in load for s in r['stats']] +peak_cpu = max(sum(float(c['CPUPerc'].strip('%')) for c in s['containers']) for s in snapshots) +peak_memory = max(sum(mib(c['MemUsage'].split('/')[0]) for c in s['containers']) for s in snapshots) +text.extend(['', f'В собранных snapshots максимальная сумма CPU контейнеров: **{peak_cpu:.1f}%** (100% = один CPU); сумма cgroup memory: **{peak_memory:.1f} MiB**. Это максимум редких снимков, не непрерывно измеренный пик.', '']) +points = ('before_publish','after_publish','before_effect','after_original','after_variants','after_commit') +text.extend(['| Точка crash | Повторов | Полная длительность сценария, s (min–max) |', '| --- | --- | --- |']) +for point in points: + rows = [r for r in faults if r['scenario'] == point] + text.append(f"| {point} | {len(rows)} | {min(r['seconds'] for r in rows):.3f}–{max(r['seconds'] for r in rows):.3f} |") +text.extend(['', 'Во всех сохранённых crash/version/replay-сценариях проверки инвариантов прошли. Увеличение входного изображения снижает throughput и повышает время до ready в измеренном профиле. Повторы показывают разброс, а не статистически надёжную оценку хвостов: выборка и длительность малы.']) +path = ROOT / 'REPORT.md' +report = path.read_text(encoding='utf-8') +begin, end = '', '' +block = begin + '\n\n' + '\n'.join(text) + '\n\n' + end +if begin in report: + report = report[:report.index(begin)] + block + report[report.index(end)+len(end):] +else: + report = report.replace('', block) +path.write_text(report, encoding='utf-8') +with (ROOT / 'evidence/measurements.csv').open('w', newline='', encoding='utf-8') as target: + writer = csv.DictWriter(target, fieldnames=['profile','repeat','completed','successful_rps','p50','p95','p99','error_fraction']) + writer.writeheader() + writer.writerows({key:row[key] for key in writer.fieldnames} for row in load) +print('\n'.join(text)) diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 00000000..a4b5173f --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,44 @@ +"""Явная готовность API, stores и процессов; smoke проверяет обработку задания.""" +import argparse +import json +import subprocess +import time +import urllib.error +import urllib.request + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--project', default='lyakhonov-lab5') + parser.add_argument('--base-url', default='http://127.0.0.1:8085') + args = parser.parse_args() + deadline = time.monotonic()+120 + observed = set() + while time.monotonic() < deadline: + ids = subprocess.check_output(['docker','compose','-p',args.project,'ps','-q'], text=True).split() + if not ids: + time.sleep(1) + continue + rows = json.loads(subprocess.check_output(['docker','inspect',*ids], text=True)) + services = {} + for row in rows: + name = row['Config']['Labels']['com.docker.compose.service'] + services.setdefault(name, []).append(row['State']) + running = all(services.get(name) and all(s['Running'] for s in services[name]) + for name in ('api','db','s3','broker','dispatcher','worker','staging-cleaner','lb')) + healthy = all(services.get(name) and all(s.get('Health',{}).get('Status') == 'healthy' for s in services[name]) + for name in ('db','broker','api')) + try: + with urllib.request.urlopen(args.base_url+'/health/ready', timeout=2) as response: + observed.add(response.headers.get('X-Instance-Id')) + except (urllib.error.URLError, TimeoutError, OSError): + pass + if running and healthy and len(services.get('api', [])) >= 2 and len(observed - {None}) >= 2: + print('PASS: two ready API instances, healthy DB/broker, required background containers running') + return + time.sleep(.5) + raise SystemExit('Readiness deadline exceeded') + + +if __name__ == '__main__': + main() diff --git a/student-tests/load_messages.py b/student-tests/load_messages.py new file mode 100644 index 00000000..ec772c6a --- /dev/null +++ b/student-tests/load_messages.py @@ -0,0 +1,22 @@ +"""Публичная нагрузка сообщений с отдельными cgroup CPU/memory snapshots.""" +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +command = [sys.executable, '-X', 'utf8', 'tests/load.py', '--base-url', 'http://127.0.0.1:8085', + '--duration', '30', '--concurrency', '4', '--output', 'evidence/load-messages.json'] +with (ROOT / 'evidence/load-messages-run.txt').open('w', encoding='utf-8') as output: + process = subprocess.Popen(command, cwd=ROOT, stdout=output, stderr=subprocess.STDOUT) + samples = [] + start = time.monotonic() + while process.poll() is None: + ids = subprocess.check_output(['docker', 'compose', '-p', 'lyakhonov-lab5', 'ps', '-q'], cwd=ROOT, text=True).split() + stats = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids], text=True) + samples.append(dict(at_seconds=round(time.monotonic()-start, 3), containers=[json.loads(x) for x in stats.splitlines()])) + time.sleep(2) + code = process.wait() +(ROOT / 'evidence/load-messages-resources.json').write_text(json.dumps(dict(command=command, exit_code=code, samples=samples), indent=2), encoding='utf-8') +raise SystemExit(code) diff --git a/student-tests/load_pipeline.py b/student-tests/load_pipeline.py index 138568a2..8fc3e411 100644 --- a/student-tests/load_pipeline.py +++ b/student-tests/load_pipeline.py @@ -51,7 +51,7 @@ def main(args): c = Client(args.base_url) begin = time.monotonic() att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png', 202, - raw=raw, actor=alice, headers={'Content-Type': 'image/png'})['json'] + raw=raw, actor=alice, headers={'Content-Type': 'image/png'}, schema='Attachment')['json'] accepted = time.monotonic() ready = c.wait_ready(alice, att) check(len(ready['variants']) == 3 and ready['sha256'] == hashlib.sha256(raw).hexdigest(), 'bad load result') diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py index 3a6f6be5..ee1f46f9 100644 --- a/student-tests/pipeline.py +++ b/student-tests/pipeline.py @@ -212,6 +212,7 @@ class Experiment: upload(huge, 413) att = self.upload() self.invariant(att) + check(self.probe('anonymous', id=att['id']) == 403, 'bucket allows anonymous reads') self.c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=self.eve) self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=self.eve) key = os.urandom(16).hex() diff --git a/student-tests/probe.py b/student-tests/probe.py index 10b3cab5..48674199 100644 --- a/student-tests/probe.py +++ b/student-tests/probe.py @@ -10,6 +10,17 @@ from src.storage import storage def run(args): action = args['action'] + if action == 'anonymous': + import boto3 + from botocore import UNSIGNED + from botocore.config import Config + from botocore.exceptions import ClientError + s3 = boto3.client('s3', endpoint_url='http://s3:9000', config=Config(signature_version=UNSIGNED)) + try: + s3.get_object(Bucket=BUCKET, Key=f"originals/{args['id']}") + except ClientError as exc: + return exc.response['ResponseMetadata']['HTTPStatusCode'] + return 200 if action == 'sql': with storage.pool.connection() as conn: cursor = conn.execute(args['query'], args.get('params', [])) diff --git a/student-tests/resilience_driver.py b/student-tests/resilience_driver.py index aed561af..62fc51ae 100644 --- a/student-tests/resilience_driver.py +++ b/student-tests/resilience_driver.py @@ -58,11 +58,11 @@ def targeted(): errors.append(str(exc)) time.sleep(.05) check(first is not None and first <= 10, 'slow API failover') - page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice)['json'] + page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice, schema='MessagePage')['json'] messages = list(page['items']) while page['next_cursor']: from urllib.parse import quote - page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe=''), actor=alice)['json'] + page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe=''), actor=alice, schema='MessagePage')['json'] messages.extend(page['items']) found = {m['id'] for m in messages} check(all(m['id'] in found for m in responses), 'acknowledged message lost')