diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..5294b68f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +* text=auto +*.sh text eol=lf +*.py text eol=lf +*.sql text eol=lf +*.yaml text eol=lf +*.conf text eol=lf +*.cmd text eol=crlf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..4e389d7a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11.10-slim-bookworm@sha256:840e180ebcc6e5c8efab209c43f5e40fd2af98cb49db5c7103c90539c56bb30e +WORKDIR /app +# Приложение работает без прав root. +RUN groupadd -r appuser && useradd -r -g appuser appuser +COPY requirements.lock.txt . +RUN pip install --no-cache-dir -r requirements.lock.txt +COPY src/ src/ +COPY migrations/ migrations/ +USER appuser +HEALTHCHECK --interval=2s --timeout=2s --start-period=10s --retries=3 CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready',timeout=1)" +EXPOSE 8080 +CMD ["python", "-m", "uvicorn", "src.app:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "8", "--timeout-keep-alive", "2", "--no-access-log"] diff --git a/Dockerfile.s3 b/Dockerfile.s3 new file mode 100644 index 00000000..5ec09f31 --- /dev/null +++ b/Dockerfile.s3 @@ -0,0 +1,17 @@ +# Build pinned upstream releases and verify archive checksums. +FROM golang:1.24.6-bookworm@sha256:ab1d1823abb55a9504d2e3e003b75b36dbeb1cbcc4c92593d85a84ee46becc6c AS build +WORKDIR /build +ADD --checksum=sha256:8819e3e7817e46b7b3798f8f200ead208562e571563c2e040352378031abe9f2 https://github.com/minio/minio/archive/07c3a429bfed433e49018cb0f78a52145d4bedeb.tar.gz /tmp/minio.tar.gz +RUN tar -xzf /tmp/minio.tar.gz --strip-components=1 && CGO_ENABLED=0 go build -trimpath -o /out/minio . +WORKDIR /build-mc +ADD --checksum=sha256:526ec538e7f83e4bfa631788129b08ec85898ca572ce0523ebf9a597946bea27 https://github.com/minio/mc/archive/refs/tags/RELEASE.2025-04-16T18-13-26Z.tar.gz /tmp/mc.tar.gz +RUN tar -xzf /tmp/mc.tar.gz --strip-components=1 && CGO_ENABLED=0 go build -trimpath -o /out/mc . +FROM debian:bookworm-20251020-slim@sha256:78d2f66e0fec9e5a39fb2c72ea5e052b548df75602b5215ed01a17171529f706 +LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.revision="07c3a429bfed433e49018cb0f78a52145d4bedeb" +COPY --from=build /build/LICENSE /usr/share/doc/minio/LICENSE +COPY --from=build /build-mc/LICENSE /usr/share/doc/mc/LICENSE +COPY --from=build /out/minio /out/mc /usr/bin/ +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY infra/s3-entrypoint.sh /usr/local/bin/s3-entrypoint.sh +ENTRYPOINT ["/bin/sh","/usr/local/bin/s3-entrypoint.sh"] +CMD ["server","/data"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..7c21e161 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,207 @@ -# Отчёт — лабораторная 5 +# Лабораторная №5 — очереди и асинхронная обработка изображений -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Ляхонов Сергей, БВТ2301. 20 ноября 2026 года.** Цель: **5**, выбран **трек 2 — доказанная идемпотентность versioned pipeline**. Ветка `solution/BVT2301-lyakhonov`. Проверенная ревизия — в [reproduction.json](evidence/reproduction.json), авторство — в [audit.json](evidence/audit.json). ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Desktop с Linux-контейнерами и Compose v2, Python 3.11+ для собственных генераторов (публичным тестам достаточно 3.9+). Проверено: Windows, Python хоста 3.13.7, Python контейнера 3.11.10, Engine 28.5.1, Compose 2.40.3-desktop.1, Intel i5-1340P, 16 логических CPU/16 GB RAM. Docker VM: 16 CPU/8 126 664 704 bytes RAM. ID образов, контейнеров, лимиты и параметры эксперимента: [environment.json](evidence/environment.json). -## Архитектура и решения +PowerShell из корня №5: -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```powershell +$env:PYTHONUTF8='1' +python scripts/bootstrap.py +docker compose -p lyakhonov-lab5 config --quiet +docker compose -p lyakhonov-lab5 build api s3 +docker compose -p lyakhonov-lab5 up -d +python scripts/wait_ready.py +python scripts/check_contract.py +python tests/smoke.py --base-url http://127.0.0.1:8085 +python -m venv .venv +.venv/Scripts/python -m pip install -r requirements-dev.txt +.venv/Scripts/python scripts/validate_openapi.py +docker compose -p lyakhonov-lab5 run --rm --no-deps -v "${PWD}:/suite:ro" -w /suite api python -m unittest discover -s student-tests -p 'test_*.py' +python student-tests/resilience_driver.py +``` -## Доказательства критериев +`make check validate test PYTHON=python BASE_URL=http://127.0.0.1:8085` эквивалентен прямым командам при наличии make и validator. UTF-8 mode обязателен на этом Windows: без него исходный публичный checker получает `UnicodeDecodeError` при чтении UTF-8 JSON как cp1251. Checker не изменён. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +Bootstrap создаёт **только отсутствующие** `.env` и `secrets/`, не перезаписывает настройки. Для чистого экспорта генерируются новые секреты. `migrate` применяет 001–006 в транзакции под advisory lock до запуска API. Повторные migrate, bucket-init и объявления очереди безопасны. Секреты передаются файлами `/run/secrets`, исключены из Git и build context. + +Рабочая конфигурация — `compose.yaml` и `lab5.env.example`. Исходный `.env.example` задания сохранён: сессии фактически в PostgreSQL, кэш через внутренний `CACHE_URL`, storage credentials отображены на `S3_ACCESS_KEY_FILE/S3_SECRET_KEY_FILE`, bucket — `attachments`. Лимиты bytes/pixels и 24h TTL идемпотентности фиксированы контрактом в коде. `WORKER_CONCURRENCY` — число worker-контейнеров, каждый имеет prefetch=1. + +Единственный опубликованный порт — `127.0.0.1:8085` HAProxy. При занятости задайте свободный `LAB5_PORT` и соответствующий URL тестов. Внутренние API, DB, broker, Redis, S3 не опубликованы. Тома/сеть принадлежат отдельному проекту `lyakhonov-lab5`, образы — `lyakhonov-messenger:lab5`, `lyakhonov-storage:lab5`. + +MinIO/mc собираются из официальных исходников: MinIO commit `07c3a429bfed433e49018cb0f78a52145d4bedeb`, mc `RELEASE.2025-04-16T18-13-26Z`. Архивы проверяются `ADD --checksum`, Go/Debian закреплены digest, Go modules — go.sum. Лицензии включены в образ. Сборка действительно прошла с использованием build cache; повторной недоступности официального MinIO-образа не утверждаю. Без кэша нужен доступ к Docker Hub/GitHub/Python package index/Go proxy. + +Остановка с сохранением данных: `docker compose -p lyakhonov-lab5 down`. Только при намеренном удалении своих учебных данных: `docker compose -p lyakhonov-lab5 down --volumes`. В persistence/failover volumes не удаляются. POSIX: python3, `.venv/bin/python`, executable `scripts/restart.sh`/`stop-one.sh`; Windows использует `.cmd` hooks. + +## Архитектура и перенос + +```mermaid +flowchart LR + C[Клиент] --> LB[HAProxy :8085] + LB --> A[2 FastAPI API] + A --> DB[(PostgreSQL: metadata, sessions, messages, jobs/outbox)] + A --> S[(MinIO: staging и final)] + D[Dispatcher: publish confirm, recovery] --> DB + D --> Q[(RabbitMQ durable queue)] + Q --> W[2 worker, prefetch 1] + W --> S + W --> DB + SC[Staging cleaner] --> DB + SC --> S +``` + +Из собственной №4 перенесены мессенджер, Argon2id/серверные сессии, keyset-пагинация, транзакционная идемпотентность сообщений, Redis-кэш профилей, приватный S3, HAProxy и рецепты сборки. История №4 не копировалась. Материалы №5, contracts, публичные tests/fixtures/examples сохранены. Существующие коммиты временного клона №5 `5894da9`, `a0ec26d`, `e376ede` перенесены без переписывания. Первоначальные незакоммиченные файлы сохранены отдельной копией вне проекта до переключения ветки. + +Изменения: upload `201 ready` → `202 queued`, устойчивый staging, jobs/outbox, отдельные broker/dispatcher/worker/cleaner, lease/retry/replay, versioned manifests. Reconciler исследования №4 не является обязательной частью №5; результаты №4 не выдаются за новые. №4 использована только для чтения: [fingerprint](evidence/lab4-baseline.json), [аудит](evidence/audit.json). + +| Роль/сервис | Версия | Готовность и данные | CPU/RAM | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| `api` ×2 | Python 3.11.10, FastAPI 0.115.0 | readiness PostgreSQL+S3; без локального durable state | 0.5/640 MiB каждый | +| `db` | PostgreSQL 16.6-alpine | pg_isready, postgres_data | 1/512 MiB | +| `s3` | закреплённый MinIO commit | bucket-init, s3_data | 1/768 MiB | +| `broker` | RabbitMQ 4.1.0-alpine | diagnostics ping, broker_data | 0.75/512 MiB | +| `worker` ×2 | приложение, Pillow 12.0.0, pika 1.3.2 | отдельный процесс, manual ack, restart unless-stopped | 0.75/768 MiB каждый | +| `dispatcher` | приложение | publisher confirm, recovery deadlines | 0.5/256 MiB | +| `staging-cleaner` | приложение | цикл 30 s, locks и реестр | 0.25/256 MiB | +| `redis` | Redis 7.4.1-alpine | необязательный восстанавливаемый кэш | 0.25/128 MiB | +| `lb` | HAProxy 3.0.6-alpine | HTTP readiness backend, round-robin | 0.5/128 MiB | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Сумма runtime-лимитов: **6.75 CPU/5376 MiB**, плюс одноразовые migrate/bucket-init и генератор. Это лимиты cgroup, не фактическое потребление. Полные зависимости закреплены в `requirements.lock.txt`. Graceful shutdown API — 8 s внутри 12 s Docker grace, worker — до 95 s; crash-тесты намеренно обходят graceful shutdown. -## Эксперименты и отказы +## Границы надёжности и инварианты -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +1. До 202 API принимает и проверяет **все** байты, пишет `staging/{attachment_id}` в приватный persistent S3; затем одной PostgreSQL-транзакцией создаёт attachment и job/outbox. Ответ — queued snapshot после COMMIT. Разрыв staging/DB может оставить неподтверждённый orphan; он не считается принятой задачей. +2. Dispatcher резервирует committed job с durable deadline 15 s, публикует ID/version/attempt/time, ждёт publisher confirm и отмечает публикацию. Смерть до publish или до отметки восстанавливается deadline и допускает повторную доставку. Байтов/credentials/произвольных URL в broker нет. +3. Логическая работа уникальна по `(attachment_id,pipeline_version)`, эффект по `(attachment_id,pipeline_version,variant)`. Claim блокирует attachment, затем job, сверяет envelope/attempt/target, выдаёт fencing token и lease. Дубль ready/processing или устаревшая версия не создаёт новую попытку. Durable lease восстанавливает работу владельца даже если другой дубль уже ack-нут. +4. Worker проверяет SHA-256/size staging, выполняет Pillow в отдельном subprocess, пишет immutable original и versioned variants с `If-None-Match: *`. Существующий key должен совпадать по metadata SHA-256/size/MIME. Manifest и ready публикуются одной DB-транзакцией только при актуальных token, lease и target version; затем ack. +5. Upgrade атомарно повышает target version и помечает старые активные jobs superseded. Старый живой worker может записать только keys своей версии и не может заменить новый manifest. Уже опубликованная v1 остаётся доступной до COMMIT v2; неуспешный upgrade виден в failed-реестре, прежний ready остаётся доступным. +6. Инварианты тестов: принятые задачи завершаются после исследованных сбоев; 3 manifest-записи на опубликованную версию; original/варианты скачиваются и совпадают с SHA-256; дубль не размножает keys; old job и old live worker не меняют published version/bytes v2. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +Это **at-least-once**, не exactly-once доставка. Повторяются доставка, resize и PUT; единственен логический опубликованный эффект. Publisher confirm — приём брокером, consumer ack — обработка доставки. Общей транзакции PostgreSQL/S3/RabbitMQ нет. + +Версии `src/pipeline_versions.py`: v1 JPEG quality=85/WebP=82, v2 JPEG=90/WebP=88. Байты вариантов действительно отличаются; original неизменен. Thumbnail вписан в 256×256 без увеличения, floor/minimum 1 px; crop — центрированный JPEG 128×128; optimized — WebP в 1280×1280 без увеличения, method=4. EXIF orientation применяется, метаданные удаляются, alpha компонуется на белый. Уменьшение каждого маленького файла не обещается. + +Ограничения: ≤10 MiB, ≤40 MP до полного decode, один upload/API-процесс, приём ≤20 s; worker subprocess timeout 60 s и worker cgroup 768 MiB. Lease по умолчанию 90 s, минимум timeout+5 s; prefetch=1. Для коротких fault-экспериментов использованы timeout=5 s/lease=10 s: [fault-environment.json](evidence/fault-environment.json). Их время восстановления нельзя объявлять временем production-конфигурации. + +До 5 управляемых попыток. Backoff `min(30,2^(attempt-1))*uniform(0.75,1.25)` s. Invalid image/integrity/object conflict постоянны; S3/timeout временны; исчерпание — `retry_exhausted`. Failed-реестр хранится в PostgreSQL, отдельная DLQ не нужна. Replay проверяет доступность source, записывает audit и выдаёт новый ограниченный бюджет. Ready replay не выполняет. + +```powershell +docker compose -p lyakhonov-lab5 exec -T --index 1 worker python -m src.operator failed +docker compose -p lyakhonov-lab5 exec -T --index 1 worker python -m src.operator replay ATTACHMENT_UUID +docker compose -p lyakhonov-lab5 exec -T --index 1 worker python -m src.operator upgrade ATTACHMENT_UUID --version 2 +``` + +Это локальные операторские расширения, обязательные HTTP endpoint/схемы не изменены. Staging ready очищается после безопасного manifest; failed/orphan retention 86400 s, для failed отсчёт от settled_at. Queued/processing не удаляются. Cleanup и upload/replay/upgrade согласованы advisory locks. Финальные original и старые versioned variants автоматически не удаляются. + +## Проверки и доказательства + +| Критерий | Команда/сценарий | Наблюдение | Доказательство | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Материалы/контракт | checker, validator, audit | структура, OpenAPI 3.1/JSON Schema, исходные материалы не изменены | [structure](evidence/structure.txt), [validation](evidence/validation.txt), [audit](evidence/audit.json) | +| Минимум №4 с новым upload | public smoke | auth, messages, pagination, ACL, идемпотентность, 202→ready, original/thumbnail | [smoke](evidence/smoke.txt) | +| Durable handoff/broker/worker | worker stop → upload/message → API+broker restart → worker start | тот же attachment ID доходит до ready | [pipeline](evidence/pipeline.json), [environment](evidence/environment.json) | +| Состояния и варианты | unit/negative/durable suite | queued/processing/ready/failed, реальные JPEG/WebP и checksums | [unit](evidence/unit.txt), [pipeline](evidence/pipeline.json) | +| Bytes/pixels/MIME/name/ACL | negative suite | 413/415/400/404, 24 конкурентных POST одного ключа дают 1 message ID | [pipeline](evidence/pipeline.json) | +| Outbox COMMIT→publish | before_publish ×3 | смерть dispatcher, автоматическое ready без ручной новой job | [pipeline](evidence/pipeline.json) | +| Publish→confirm bookkeeping | after_publish ×3 | смерть до DB-отметки, повтор безопасен | [pipeline](evidence/pipeline.json) | +| Lease/side effect/ack | before_effect, after_original, after_variants, after_commit ×3 | os._exit(91), restart; те же bytes и 3 manifest-записи | [pipeline](evidence/pipeline.json) | +| Retry/failed/replay | remove/corrupt staging | постоянный отказ за 1 попытку, временный — 5, replay с audit до ready | [pipeline](evidence/pipeline.json) | +| Очистка staging | cleanup после ready → upgrade | v2 построена из immutable original после очистки | [pipeline](evidence/pipeline.json) | +| Трек 2: duplicate/reorder/stale | version suite ×3 | 52 явно опубликованных сообщения/повтор, старая paused попытка освобождена после v2, новый результат сохранён | [pipeline](evidence/pipeline.json) | +| Persistence stores | resilience_driver | публичный persistence и реальные StartedAt API/DB/S3/broker/Redis | [persistence](evidence/persistence.json) | +| Failover API | resilience_driver | публичный failover + отдельная остановка каждой API, сохранность подтверждений | [public](evidence/failover.json), [каждая API](evidence/failover-each.json) | +| Нагрузка полного pipeline | load_pipeline | completed images, ready latency, ошибки, CPU/memory | [load](evidence/load-pipeline.json) | +| Чистое воспроизведение | scripts/reproduce.py | Git export, новые secrets/images/project/volumes/port, build+smoke, cleanup своих временных ресурсов | [reproduction](evidence/reproduction.json) | -## Ограничения и следующие шаги +Fault hooks выключены по умолчанию, включаются только окружением `LAB5_FAULT_INJECTION=1`, не через HTTP. `experiment_faults` фиксирует arm/hit/process_id. Crash — реальная смерть процесса без finally/ack. Pause удерживает старую попытку до публикации новой. Каждый arm одноразовый. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Одноразовый `inspector` профиля `tools` нужен только операторскому тесту списка S3-объектов; он не запускается обычным `up` и не публикует порты. Root credentials монтируются только в этот одноразовый процесс и bucket-init. Приложению список final-prefix не разрешён: первый запуск тестового probe получил `AccessDenied`, что сохранено в `pipeline-initial-run.txt`. Инспектор исправлен без расширения policy приложения; crash-результаты сохранены, version/error suite продолжен с `--resume`. + +```powershell +$env:LAB5_FAULT_INJECTION='1' +$env:JOB_TIMEOUT_SECONDS='5' +$env:JOB_LEASE_SECONDS='10' +docker compose -p lyakhonov-lab5 up -d +python scripts/wait_ready.py +python student-tests/pipeline.py --repeats 3 +Remove-Item Env:LAB5_FAULT_INJECTION,Env:JOB_TIMEOUT_SECONDS,Env:JOB_LEASE_SECONDS +docker compose -p lyakhonov-lab5 up -d +python scripts/wait_ready.py +python student-tests/load_pipeline.py --duration 20 --concurrency 2 --repeats 3 +python tests/load.py --base-url http://127.0.0.1:8085 --duration 30 --concurrency 4 --output evidence/load-messages.json +python scripts/environment.py +python scripts/reproduce.py +python scripts/audit.py +``` + +## Методика и результаты + +Гипотеза: fencing публикации manifest и immutable versioned S3 keys сохраняют принятые задачи и результат новой версии при дублях, reorder и процессных crash/restart в исследованной модели. + +Fault suite использует fixture 320×200, 111817 bytes. Шесть окон crash ×3 повтора. График показывает **полную длительность сценария** (stop, upload, start, ready, downloads), не чистую latency восстановления. Сохранены ID/StartedAt и состояния до/после, checksums всех четырёх скачанных результатов. + +![Реальные crash-сценарии](evidence/faults.png) + +Нагрузка closed-loop: 2 клиента, 20 s запуска операций, один завершённый warmup/профиль/повтор, 3 повтора. Профили: публичный PNG 320×200 и детерминированный RGB PNG 1024×768 с псевдослучайными пикселями. Клиент ждёт **ready и три варианта**, затем начинает следующий upload. Знаменатель successful RPS — фактическая длительность с завершением последней операции. Ошибки сохраняются, прозрачных retry upload нет. Длительность включает передачу bytes, очередь, обработку и polling. + + + +| Профиль | Bytes/image | Завершено за 3 повтора | Успешных image/s (min–max) | p50 ready, s | p95 ready, s | p99 ready, s | Ошибки | +| --- | --- | --- | --- | --- | --- | --- | --- | +| 1024x768 | 2360847 | 117 | 1.569–2.188 | 0.790–1.067 | 1.140–1.230 | 1.161–1.322 | 1 | +| 320x200 | 111817 | 224 | 3.549–3.982 | 0.401–0.439 | 0.727–0.769 | 0.744–0.820 | 0 | + +В собранных snapshots максимальная сумма CPU контейнеров: **218.3%** (100% = один CPU); сумма cgroup memory: **1047.5 MiB**. Это максимум редких снимков, не непрерывно измеренный пик. + +| Точка crash | Повторов | Полная длительность сценария, s (min–max) | +| --- | --- | --- | +| before_publish | 3 | 28.940–32.205 | +| after_publish | 3 | 14.478–16.650 | +| before_effect | 3 | 25.678–28.623 | +| after_original | 3 | 25.642–26.509 | +| after_variants | 3 | 25.458–27.131 | +| after_commit | 3 | 14.989–17.081 | + +Во всех сохранённых crash/version/replay-сценариях проверки инвариантов прошли. Увеличение входного изображения снижает throughput и повышает время до ready в измеренном профиле. Повторы показывают разброс, а не статистически надёжную оценку хвостов: выборка и длительность малы. + + + +![Throughput и p95 полного pipeline](evidence/load.png) + +`docker stats --no-stream` собирается примерно раз в 2–4 s параллельно нагрузке: CPU%, cgroup memory/limits по контейнерам. Редкие snapshots могут пропустить пики и **не являются точным RSS**. Генератор на том же ноутбуке; polling 0.3 s добавляет квантование/coordinated omission. Это не доказательство максимальной производительности/SLA. Дополнительный публичный POST-message load — [load-messages.json](evidence/load-messages.json). + +В профиле 1024×768 во втором повторе сохранена **одна ошибка GET соединения `URLError`** (1/33 наблюдавшихся операций, 3.03% этого повтора). Её причина не установлена: публичный helper не сохраняет низкоуровневую причину URLError. Повтор ради удаления ошибки из статистики не выполнялся. Вложение `63d3337b-5ac0-478b-9e30-cde11dd2fd07`, для которого клиент уже получил 202, отдельно найдено ready с актуальным manifest: [final-state.json](evidence/final-state.json). Финальная проверка всех вложений обнаружила ноль нарушений готовности/версии/числа вариантов. Ошибка наблюдения учтена в измерении и не включена в успешный RPS. + +Публичная нагрузка сообщений: 4 клиента, 30.112 s, 2507 успешных POST, 0 ошибок, **83.256 successful RPS**, p50/p95/p99 успешных запросов **25.341/146.956/251.292 ms**. Это один дополнительный запуск, не серия для оценки максимума; CPU/memory snapshots — [load-messages-resources.json](evidence/load-messages-resources.json). + +Графики и таблица воспроизводятся из агрегатов: + +```powershell +.venv/Scripts/python -m pip install -r requirements-research.txt +.venv/Scripts/python scripts/plot_results.py +python scripts/summarize.py +python scripts/final_state.py +``` + +Matplotlib 3.10.7 нужен только для графиков. Вспомогательный targeted-failover driver был исправлен после ошибки чтения JSON без указания schema публичному Client; сервер не менялся, публичные persistence/failover уже прошли, обе остановки затем проверены исправленным driver. Первичный вывод сохранён в `resilience-initial-run.txt`. В этой версии Compose `up --wait` ошибочно потребовал healthcheck отключённых background services; запуск использует `up -d` и явный `scripts/wait_ready.py`, после которого smoke проверяет реальную обработку. + +## Ограничения + +Проверены процессные crash/restart с сохранёнными volumes на одном Docker-host. Не исследованы потеря диска/host, порча опубликованных объектов, византийские компоненты или multi-node split-brain. DB/S3/broker/LB одиночные. Durable classic queue одного брокера не даёт multi-node HA. После confirm предполагается сохранность broker volume; полная потеря этого volume вне модели. + +Нет общей транзакции PostgreSQL/S3. Неподтверждённая загрузка может оставить staging orphan до cleanup. Объекты superseded версии могут остаться недоступными через API; их GC не реализован. Повтор immutable PUT доверяет metadata SHA-256/size/MIME существующего object; runtime не перечитывает все его bytes. Тесты отдельно скачивают и пересчитывают bytes, но намеренная внешняя порча после ready этим pipeline не исправляется. + +Cgroup ограничивает весь worker, а не гарантирует обработку любого допустимого 40-MP изображения: OOM может убить worker, затем lease/retry завершают ready либо failed. Subprocess не является полной security sandbox. Стенд с синтетическими данными предназначен для loopback. + +## Источники и ИИ + +- Нормативные README/COURSE/contracts/tests текущей №5 и собственная база №4; результаты измерены заново. +- [RabbitMQ confirms/acknowledgements](https://www.rabbitmq.com/docs/confirms): независимые границы подтверждения. +- [S3 conditional writes](https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-writes.html): If-None-Match; совместимость закреплённого MinIO проверена повторными записями. +- [Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html): EXIF transpose/fit; фактическая версия 12.0.0 в lockfile. +- [MinIO source](https://github.com/minio/minio/tree/07c3a429bfed433e49018cb0f78a52145d4bedeb), [mc source](https://github.com/minio/mc/tree/RELEASE.2025-04-16T18-13-26Z). + +Codex использован для анализа, переноса сохранённой работы, доработки кода, сценариев, запуска экспериментов и отчёта. PASS относится только к фактически выполненным проверкам. Автор должен понимать и уметь объяснить решение на защите. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..a30fb4fc --- /dev/null +++ b/compose.yaml @@ -0,0 +1,177 @@ +name: lyakhonov-lab5 + +x-app: &app + image: lyakhonov-messenger:lab5 + build: . + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + LAB5_FAULT_INJECTION: ${LAB5_FAULT_INJECTION:-0} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + CACHE_URL: redis://redis:6379/0 + BROKER_URL_FILE: /run/secrets/broker_url + secrets: [database_url, s3_access_key, s3_secret_key, broker_url] + cpus: 0.5 + mem_limit: 640m + stop_grace_period: 12s + +x-dependencies: &dependencies + migrate: + condition: service_completed_successfully + bucket-init: + condition: service_completed_successfully + +services: + inspector: + <<: *app + profiles: [tools] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_ACCESS_KEY_FILE: /run/secrets/s3_root_user + S3_SECRET_KEY_FILE: /run/secrets/s3_root_password + secrets: [database_url, s3_root_user, s3_root_password] + healthcheck: + disable: true + cpus: 0.25 + mem_limit: 256m + db: + image: postgres:16.6-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 30 + cpus: 1.0 + mem_limit: 512m + redis: + image: redis:7.4.1-alpine + command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] + cpus: 0.25 + mem_limit: 128m + s3: + image: lyakhonov-storage:lab5 + build: + context: . + dockerfile: Dockerfile.s3 + command: [server, /data, --console-address, '127.0.0.1:9001'] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password + secrets: [s3_root_user, s3_root_password] + volumes: [s3_data:/data] + cpus: 1.0 + mem_limit: 768m + bucket-init: + image: lyakhonov-storage:lab5 + pull_policy: never + entrypoint: [/bin/sh, /bootstrap/bucket.sh] + secrets: [s3_root_user, s3_root_password, s3_access_key, s3_secret_key] + volumes: [./infra/bucket.sh:/bootstrap/bucket.sh:ro, ./infra/s3-policy.json:/bootstrap/policy.json:ro] + depends_on: [s3] + cpus: 0.25 + mem_limit: 128m + broker: + image: rabbitmq:4.1.0-alpine + hostname: broker + entrypoint: [/bin/sh, /bootstrap/broker.sh] + volumes: [broker_data:/var/lib/rabbitmq, ./infra/broker.sh:/bootstrap/broker.sh:ro, + ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-lab5.conf:ro] + secrets: [broker_password] + healthcheck: + test: [CMD, rabbitmq-diagnostics, -q, ping] + interval: 5s + timeout: 4s + retries: 20 + cpus: 0.75 + mem_limit: 512m + migrate: + <<: *app + command: [python, -m, src.migrate] + depends_on: + db: + condition: service_healthy + healthcheck: + disable: true + api: + <<: *app + depends_on: *dependencies + deploy: + replicas: ${API_REPLICAS:-2} + dispatcher: + <<: *app + command: [python, -m, src.dispatch] + restart: unless-stopped + depends_on: + <<: *dependencies + broker: + condition: service_healthy + healthcheck: + disable: true + mem_limit: 256m + worker: + <<: *app + command: [python, -m, src.worker] + restart: unless-stopped + depends_on: + <<: *dependencies + broker: + condition: service_healthy + healthcheck: + disable: true + deploy: + replicas: ${WORKER_CONCURRENCY:-2} + cpus: 0.75 + mem_limit: 768m + stop_grace_period: 95s + staging-cleaner: + <<: *app + command: [python, -m, src.staging] + restart: unless-stopped + depends_on: *dependencies + healthcheck: + disable: true + cpus: 0.25 + mem_limit: 256m + lb: + image: haproxy:3.0.6-alpine + ports: ['127.0.0.1:${LAB5_PORT:-8085}:8080'] + volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro] + depends_on: [api] + cpus: 0.5 + mem_limit: 128m + stop_grace_period: 12s + +secrets: + s3_root_user: + file: ./secrets/s3_root_user + s3_root_password: + file: ./secrets/s3_root_password + s3_access_key: + file: ./secrets/s3_access_key + s3_secret_key: + file: ./secrets/s3_secret_key + db_password: + file: ./secrets/db_password + database_url: + file: ./secrets/database_url + broker_password: + file: ./secrets/broker_password + broker_url: + file: ./secrets/broker_url +volumes: + postgres_data: + s3_data: + broker_data: diff --git a/evidence/audit.json b/evidence/audit.json new file mode 100644 index 00000000..b9c9bf8f --- /dev/null +++ b/evidence/audit.json @@ -0,0 +1,14 @@ +{ + "lab4_unchanged": true, + "protected_lab5_unchanged": true, + "tracked_local_secrets": false, + "scanned_files": 132, + "authors": [ + "ef1f470|Sergey Lyakhonov|lyserg119@gmail.com|Sergey Lyakhonov|lyserg119@gmail.com", + "42bf6aa|Sergey Lyakhonov|lyserg119@gmail.com|Sergey Lyakhonov|lyserg119@gmail.com", + "e376ede|Sergey Lyakhonov|lyserg119@gmail.com|Sergey Lyakhonov|lyserg119@gmail.com", + "a0ec26d|Sergey Lyakhonov|lyserg119@gmail.com|Sergey Lyakhonov|lyserg119@gmail.com", + "5894da9|Sergey Lyakhonov|lyserg119@gmail.com|Sergey Lyakhonov|lyserg119@gmail.com" + ], + "head": "ef1f470659610f8ee0dc20bc3315589ba18154ca" +} \ No newline at end of file diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 00000000..458feec1 --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,263 @@ +{ + "commit": "42bf6aa702dab9270fc13d7b80df13dfca089ef8", + "python": "3.13.7", + "working_tree_dirty": true, + "docker": { + "Client": { + "Version": "28.5.1", + "ApiVersion": "1.51", + "DefaultAPIVersion": "1.51", + "GitCommit": "e180ab8", + "GoVersion": "go1.24.8", + "Os": "windows", + "Arch": "amd64", + "BuildTime": "Wed Oct 8 12:19:16 2025", + "Context": "desktop-linux" + }, + "Server": { + "Platform": { + "Name": "Docker Desktop 4.49.0 (208700)" + }, + "Components": [ + { + "Name": "Engine", + "Version": "28.5.1", + "Details": { + "ApiVersion": "1.51", + "Arch": "amd64", + "BuildTime": "Wed Oct 8 12:17:24 2025", + "Experimental": "false", + "GitCommit": "f8215cc", + "GoVersion": "go1.24.8", + "KernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "MinAPIVersion": "1.24", + "Os": "linux" + } + }, + { + "Name": "containerd", + "Version": "1.7.27", + "Details": { + "GitCommit": "05044ec0a9a75232cad458027ca83437aae3f4da" + } + }, + { + "Name": "runc", + "Version": "1.2.5", + "Details": { + "GitCommit": "v1.2.5-0-g59923ef" + } + }, + { + "Name": "docker-init", + "Version": "0.19.0", + "Details": { + "GitCommit": "de40ad0" + } + } + ], + "Version": "28.5.1", + "ApiVersion": "1.51", + "MinAPIVersion": "1.24", + "GitCommit": "f8215cc", + "GoVersion": "go1.24.8", + "Os": "linux", + "Arch": "amd64", + "KernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "BuildTime": "2025-10-08T12:17:24.000000000+00:00" + } + }, + "compose": "Docker Compose version v2.40.3-desktop.1", + "engine": { + "memory_bytes": 8126664704, + "cpus": 16 + }, + "services": [ + { + "service": "api", + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T04:00:04.137137202Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 671088640, + "nanocpus": 500000000, + "ports": {}, + "settings": [ + "JOB_MAX_ATTEMPTS=5", + "JOB_TIMEOUT_SECONDS=60", + "STAGING_RETENTION_SECONDS=86400", + "JOB_LEASE_SECONDS=90", + "LAB5_FAULT_INJECTION=0" + ] + }, + { + "service": "api", + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T04:00:23.843924506Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 671088640, + "nanocpus": 500000000, + "ports": {}, + "settings": [ + "LAB5_FAULT_INJECTION=0", + "STAGING_RETENTION_SECONDS=86400", + "JOB_MAX_ATTEMPTS=5", + "JOB_LEASE_SECONDS=90", + "JOB_TIMEOUT_SECONDS=60" + ] + }, + { + "service": "broker", + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "image": "rabbitmq:4.1.0-alpine", + "image_id": "sha256:8a73e9185f8420247c5b3c19231df18dfac28a8d5706203e13b291934044966d", + "started_at": "2026-11-20T03:58:31.88055044Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nanocpus": 750000000, + "ports": {}, + "settings": [] + }, + { + "service": "db", + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "image": "postgres:16.6-alpine", + "image_id": "sha256:1d04b9ba1d4996401f2552b51beda8187f175c0645c091e4781134fc9c9a3eef", + "started_at": "2026-11-20T03:58:30.504381937Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 536870912, + "nanocpus": 1000000000, + "ports": {}, + "settings": [] + }, + { + "service": "dispatcher", + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:58:31.312524166Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nanocpus": 500000000, + "ports": {}, + "settings": [ + "JOB_TIMEOUT_SECONDS=60", + "STAGING_RETENTION_SECONDS=86400", + "JOB_LEASE_SECONDS=90", + "LAB5_FAULT_INJECTION=0", + "JOB_MAX_ATTEMPTS=5" + ] + }, + { + "service": "lb", + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "image": "haproxy:3.0.6-alpine", + "image_id": "sha256:c22c8710886104a48b920306f063401f0d11811858e3c6b9d87d88a7556b2e61", + "started_at": "2026-11-20T03:58:30.862509137Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 134217728, + "nanocpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "settings": [] + }, + { + "service": "redis", + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "image": "redis:7.4.1-alpine", + "image_id": "sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275", + "started_at": "2026-11-20T03:58:30.614258194Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 134217728, + "nanocpus": 250000000, + "ports": {}, + "settings": [] + }, + { + "service": "s3", + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "image": "lyakhonov-storage:lab5", + "image_id": "sha256:88795615d5dbc1a5618ff618334251801720cb8ba30ca34d65d6b0ad508a2afe", + "started_at": "2026-11-20T03:58:31.091098179Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nanocpus": 1000000000, + "ports": {}, + "settings": [] + }, + { + "service": "staging-cleaner", + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:58:31.667560755Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 268435456, + "nanocpus": 250000000, + "ports": {}, + "settings": [ + "JOB_MAX_ATTEMPTS=5", + "JOB_TIMEOUT_SECONDS=60", + "LAB5_FAULT_INJECTION=0", + "JOB_LEASE_SECONDS=90", + "STAGING_RETENTION_SECONDS=86400" + ] + }, + { + "service": "worker", + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:58:31.445500705Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nanocpus": 750000000, + "ports": {}, + "settings": [ + "STAGING_RETENTION_SECONDS=86400", + "LAB5_FAULT_INJECTION=0", + "JOB_MAX_ATTEMPTS=5", + "JOB_TIMEOUT_SECONDS=60", + "JOB_LEASE_SECONDS=90" + ] + }, + { + "service": "worker", + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:58:31.518418339Z", + "restart_count": 0, + "oom_killed": false, + "memory_limit": 805306368, + "nanocpus": 750000000, + "ports": {}, + "settings": [ + "JOB_TIMEOUT_SECONDS=60", + "LAB5_FAULT_INJECTION=0", + "JOB_MAX_ATTEMPTS=5", + "JOB_LEASE_SECONDS=90", + "STAGING_RETENTION_SECONDS=86400" + ] + } + ] +} \ No newline at end of file diff --git a/evidence/failover-each.json b/evidence/failover-each.json new file mode 100644 index 00000000..c8f1d872 --- /dev/null +++ b/evidence/failover-each.json @@ -0,0 +1,336 @@ +[ + { + "target": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "before": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:59:21.0995573Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:59:04.166647284Z", + "running": true + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:59:01.428198308Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:59:01.96581278Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ], + "after": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:59:21.0995573Z", + "running": false + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:59:04.166647284Z", + "running": true + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:59:01.428198308Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:59:01.96581278Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ], + "successes": 182, + "errors": [], + "recovery_seconds": 1.011 + }, + { + "target": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "before": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T04:00:04.137137202Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:59:04.166647284Z", + "running": true + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:59:01.428198308Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:59:01.96581278Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ], + "after": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T04:00:04.137137202Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:59:04.166647284Z", + "running": false + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:59:01.428198308Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:59:01.96581278Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ], + "successes": 176, + "errors": [ + "POST /api/v1/chats/6ba7728e-901a-4ce8-99ec-ff8a6495b5fa/messages: expected [201], received 503" + ], + "recovery_seconds": 1.932 + } +] \ No newline at end of file diff --git a/evidence/failover-targeted-run.txt b/evidence/failover-targeted-run.txt new file mode 100644 index 00000000..ae01f1c4 --- /dev/null +++ b/evidence/failover-targeted-run.txt @@ -0,0 +1 @@ +PASS: both real API containers independently stopped and restored diff --git a/evidence/failover.json b/evidence/failover.json new file mode 100644 index 00000000..ba8daa39 --- /dev/null +++ b/evidence/failover.json @@ -0,0 +1,176 @@ +{ + "command": [ + "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-X", + "utf8", + "tests/resilience.py", + "--mode", + "failover", + "--base-url", + "http://127.0.0.1:8085", + "--action-script", + "C:\\Users\\User\\Desktop\\vysoko-nagr 5\\scripts\\stop-one.cmd" + ], + "exit_code": 0, + "output": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 88, \"failed_attempts\": 4, \"first_recovery_seconds\": 2.856}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.\n", + "before": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:58:31.589874697Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:58:31.785178375Z", + "running": true + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:58:36.540744301Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:58:37.048579415Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ], + "after": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:58:31.589874697Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:58:31.785178375Z", + "running": false + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:58:36.540744301Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:58:37.048579415Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ] +} \ No newline at end of file diff --git a/evidence/fault-environment.json b/evidence/fault-environment.json new file mode 100644 index 00000000..eec0df19 --- /dev/null +++ b/evidence/fault-environment.json @@ -0,0 +1,240 @@ +{ + "commit": "e376ede0a22aca6a8b1cc80059194dd1635861e5", + "python": "3.13.7", + "docker": { + "Client": { + "Version": "28.5.1", + "ApiVersion": "1.51", + "DefaultAPIVersion": "1.51", + "GitCommit": "e180ab8", + "GoVersion": "go1.24.8", + "Os": "windows", + "Arch": "amd64", + "BuildTime": "Wed Oct 8 12:19:16 2025", + "Context": "desktop-linux" + }, + "Server": { + "Platform": { + "Name": "Docker Desktop 4.49.0 (208700)" + }, + "Components": [ + { + "Name": "Engine", + "Version": "28.5.1", + "Details": { + "ApiVersion": "1.51", + "Arch": "amd64", + "BuildTime": "Wed Oct 8 12:17:24 2025", + "Experimental": "false", + "GitCommit": "f8215cc", + "GoVersion": "go1.24.8", + "KernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "MinAPIVersion": "1.24", + "Os": "linux" + } + }, + { + "Name": "containerd", + "Version": "1.7.27", + "Details": { + "GitCommit": "05044ec0a9a75232cad458027ca83437aae3f4da" + } + }, + { + "Name": "runc", + "Version": "1.2.5", + "Details": { + "GitCommit": "v1.2.5-0-g59923ef" + } + }, + { + "Name": "docker-init", + "Version": "0.19.0", + "Details": { + "GitCommit": "de40ad0" + } + } + ], + "Version": "28.5.1", + "ApiVersion": "1.51", + "MinAPIVersion": "1.24", + "GitCommit": "f8215cc", + "GoVersion": "go1.24.8", + "Os": "linux", + "Arch": "amd64", + "KernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "BuildTime": "2025-10-08T12:17:24.000000000+00:00" + } + }, + "compose": "Docker Compose version v2.40.3-desktop.1", + "engine": { + "memory_bytes": 8126664704, + "cpus": 16 + }, + "services": [ + { + "service": "api", + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:42:15.518481555Z", + "memory_limit": 671088640, + "nanocpus": 500000000, + "ports": {}, + "settings": [ + "JOB_TIMEOUT_SECONDS=5", + "JOB_LEASE_SECONDS=10", + "JOB_MAX_ATTEMPTS=5", + "LAB5_FAULT_INJECTION=1", + "STAGING_RETENTION_SECONDS=86400" + ] + }, + { + "service": "api", + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:42:15.729148361Z", + "memory_limit": 671088640, + "nanocpus": 500000000, + "ports": {}, + "settings": [ + "JOB_TIMEOUT_SECONDS=5", + "JOB_LEASE_SECONDS=10", + "JOB_MAX_ATTEMPTS=5", + "LAB5_FAULT_INJECTION=1", + "STAGING_RETENTION_SECONDS=86400" + ] + }, + { + "service": "broker", + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "image": "rabbitmq:4.1.0-alpine", + "image_id": "sha256:8a73e9185f8420247c5b3c19231df18dfac28a8d5706203e13b291934044966d", + "started_at": "2026-11-20T03:42:16.413919787Z", + "memory_limit": 536870912, + "nanocpus": 750000000, + "ports": {}, + "settings": [] + }, + { + "service": "db", + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "image": "postgres:16.6-alpine", + "image_id": "sha256:1d04b9ba1d4996401f2552b51beda8187f175c0645c091e4781134fc9c9a3eef", + "started_at": "2026-11-20T03:39:52.15355593Z", + "memory_limit": 536870912, + "nanocpus": 1000000000, + "ports": {}, + "settings": [] + }, + { + "service": "dispatcher", + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:45:44.20441819Z", + "memory_limit": 268435456, + "nanocpus": 500000000, + "ports": {}, + "settings": [ + "STAGING_RETENTION_SECONDS=86400", + "JOB_LEASE_SECONDS=10", + "JOB_TIMEOUT_SECONDS=5", + "LAB5_FAULT_INJECTION=1", + "JOB_MAX_ATTEMPTS=5" + ] + }, + { + "service": "lb", + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "image": "haproxy:3.0.6-alpine", + "image_id": "sha256:c22c8710886104a48b920306f063401f0d11811858e3c6b9d87d88a7556b2e61", + "started_at": "2026-11-20T03:39:59.703127904Z", + "memory_limit": 134217728, + "nanocpus": 500000000, + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "settings": [] + }, + { + "service": "redis", + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "image": "redis:7.4.1-alpine", + "image_id": "sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275", + "started_at": "2026-11-20T03:39:52.151204203Z", + "memory_limit": 134217728, + "nanocpus": 250000000, + "ports": {}, + "settings": [] + }, + { + "service": "s3", + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "image": "lyakhonov-storage:lab5", + "image_id": "sha256:88795615d5dbc1a5618ff618334251801720cb8ba30ca34d65d6b0ad508a2afe", + "started_at": "2026-11-20T03:39:52.148504843Z", + "memory_limit": 805306368, + "nanocpus": 1000000000, + "ports": {}, + "settings": [] + }, + { + "service": "staging-cleaner", + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:41:58.299334822Z", + "memory_limit": 268435456, + "nanocpus": 250000000, + "ports": {}, + "settings": [ + "JOB_LEASE_SECONDS=10", + "JOB_MAX_ATTEMPTS=5", + "LAB5_FAULT_INJECTION=1", + "JOB_TIMEOUT_SECONDS=5", + "STAGING_RETENTION_SECONDS=86400" + ] + }, + { + "service": "worker", + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:45:59.755318954Z", + "memory_limit": 805306368, + "nanocpus": 750000000, + "ports": {}, + "settings": [ + "LAB5_FAULT_INJECTION=1", + "STAGING_RETENTION_SECONDS=86400", + "JOB_LEASE_SECONDS=10", + "JOB_TIMEOUT_SECONDS=5", + "JOB_MAX_ATTEMPTS=5" + ] + }, + { + "service": "worker", + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "image": "lyakhonov-messenger:lab5", + "image_id": "sha256:2153c7be14d77d6d6e0337a445cca1feb814bef892dfaae459d9a28ce31494a4", + "started_at": "2026-11-20T03:45:58.61301125Z", + "memory_limit": 805306368, + "nanocpus": 750000000, + "ports": {}, + "settings": [ + "JOB_LEASE_SECONDS=10", + "LAB5_FAULT_INJECTION=1", + "STAGING_RETENTION_SECONDS=86400", + "JOB_TIMEOUT_SECONDS=5", + "JOB_MAX_ATTEMPTS=5" + ] + } + ] +} \ No newline at end of file diff --git a/evidence/faults.png b/evidence/faults.png new file mode 100644 index 00000000..de9d7ee0 Binary files /dev/null and b/evidence/faults.png differ diff --git a/evidence/final-state.json b/evidence/final-state.json new file mode 100644 index 00000000..1177a602 --- /dev/null +++ b/evidence/final-state.json @@ -0,0 +1,35 @@ +{ + "attachments": [ + { + "state": "ready", + "count": 384 + } + ], + "jobs": [ + { + "state": "ready", + "pipeline_version": 1, + "count": 380 + }, + { + "state": "superseded", + "pipeline_version": 1, + "count": 4 + }, + { + "state": "ready", + "pipeline_version": 2, + "count": 7 + } + ], + "replays": 2, + "load_observation_errors": [ + { + "id": "63d3337b-5ac0-478b-9e30-cde11dd2fd07", + "state": "ready", + "published_pipeline_version": 1, + "target_pipeline_version": 1 + } + ], + "violations": 0 +} \ No newline at end of file diff --git a/evidence/isolation.txt b/evidence/isolation.txt new file mode 100644 index 00000000..2817a2d5 --- /dev/null +++ b/evidence/isolation.txt @@ -0,0 +1,2 @@ +PASS: negative_upload_acl_and_24_concurrent_messages +PASS: accepted_survives_api_broker_restart diff --git a/evidence/lab4-baseline.json b/evidence/lab4-baseline.json new file mode 100644 index 00000000..8eee394f --- /dev/null +++ b/evidence/lab4-baseline.json @@ -0,0 +1,126 @@ +{ + "head": "8f112e092a10d72a98c3d9d4554c52b81ebff2b3", + "branch": "solution/BVT2301-lyakhonov", + "status": "", + "files": { + ".dockerignore": "9c307884eda8387a781e8942b4df571ae0eee950c822aff044e7ed3d21e325f9", + ".env.example": "292e0f2d8d4db97ec3cd9826db8f8bd37fcf310bb27cbc341645e36de5046bdf", + ".gitattributes": "f3c03226a7748b6a723a20549727f38b6fdea151f177fd7c1901c9af7151640a", + ".github/pull_request_template.md": "416abfd7d727759a69eed876b415079816e6fcf458e261dcac78cc719709d66f", + ".github/workflows/application.yml": "0d45cb1d8fa2a45ef8a368892b833b26d300c9ccc7310bf7930c2a3fd9227164", + ".github/workflows/contracts.yml": "57acaf40811dddc83f27bc52bff3e22e60f817ef1b71d764678562ec56f5ca97", + ".gitignore": "fd2c5f54cd9dde6efd64b3977cd10d1400f83609939585778a2acce1b087581d", + "CONTRIBUTING.md": "08f247085938bd1efc4b8f7f48e3d520b1dc11ce569f4259491e8f01b9abd02c", + "COURSE.md": "1e3c65e345b7f89d08ed86a3b50061935ee8157907cb4043cfa40689781f54e0", + "Dockerfile": "c2743d8778c97cfd43d991d2a6b4a92e4560672af4c4eae86565cd1fa9c21f78", + "Dockerfile.example": "37050a40f15e29ee7f5eed7b2123fea9e6531d68b5735256e4c41c60255cffc9", + "Dockerfile.s3": "384837c64e421e67f81355cafa0af0b32157fe54451b886e471f69af8e0a0285", + "MAINTAINERS.md": "86153124dc51f97e29b9f7fd17d0a7100571bf584ae8167f162cd9728451c124", + "Makefile": "15525fe37647b398715597764a4ff6bfbcd7b46515e5a983ff013ec01dfe302a", + "README.md": "db1ff101968c21d6ada8972c1f4708936457af2fc1fa37ff7918137a83de3860", + "REPORT.md": "a5311b5fbbb1c21871ecc2a6ea16267453fc22c5c5121f03677e48524016885b", + "compose.example.yaml": "f7c1ae414a65cabb4ec334293d237bedbf8b1a694bfd7a72cfd28a2667222d7c", + "compose.yaml": "4f6ee1613db8937a39d73aa45e23a4c3d9931ae990d7e8cf503bac4bf266db6c", + "contracts/README.md": "4c04faede4a2ffa35183455b51da21764b7fa3997c0a4b3963614b82586eed28", + "contracts/examples.json": "2816da2e63058d55bdcbeceb92b231ea2509c86df8c5a0aa908be83bd0599469", + "contracts/infrastructure.json": "18a6b38deab39878da174f01e54addc4c7de556f9b3b7f37d96ec1ef0dd240b7", + "contracts/openapi.json": "61c7dbac7bd8d57423a2883b4b7b1277e8e35918d25c37e2cd02eaf6817936f0", + "evidence/README.md": "4ae3ab8da9fe3af5442f6b2e60c55d7d34de7bf0addffcc6a55f981baf0fe5cf", + "evidence/attachments.txt": "f5890fcb8a2ddc7b165633adc0a4c34100d9cc3d4261bb217af58e6aa2730a99", + "evidence/audit.txt": "d531211f514c58865dc2dfe19764cab7d54d41606fe9dc39e7e47de8aec8cb78", + "evidence/bootstrap-repeat.txt": "f7e10da821041ce9968ca4b0218c936f154ed02414443abaa4f8b1c0ab03d8c5", + "evidence/check.txt": "695ccac9f75e5d2118641ba2b3f386e916796271477d7bc5e98700d67371624d", + "evidence/clean-export.json": "34ef3e5865d6f8edd80ea14503117534da193eb43d5a5f8ff68abf28b86ac8ff", + "evidence/cpu.json": "aafd0ff20c15b4e30532823185578cdd86aa00050231e1ba72d868221a5abf44", + "evidence/docker-resources.json": "d44b532980090157607205f726b7799fd8a8da612a722a6d1a0b829618f8e1de", + "evidence/failover-1.txt": "e354ceccdbbf099a9adb922fdda0cc8576d7cbe9a3cb5a47dddd396a90559c16", + "evidence/failover-2.txt": "3cfe80f5864aa30972075db20fb2c0c02bc3dc3a74a9e4b975b816a4797b8c07", + "evidence/figures/reconcile-cost.png": "b84d7d37da4e9791941f1288f14f1155a687824a76cf3cf89a82410ed890dce9", + "evidence/figures/reconcile-cost.svg": "3c27e170f99738946899995808e5fdcb04e0ed27c090fd7c943a7592b94e20da", + "evidence/figures/upload-memory.png": "5c3a8e0e008741b67d949b1eb2b976bb684bb0fcde1bb8bd7a7e05c679c19885", + "evidence/figures/upload-memory.svg": "c45b7952a662ae6938f7299b9a9b876d08b3301aabfabe5ce496b85d7d70110c", + "evidence/figures/upload-throughput.png": "32bcd3c589d7e21a8b6d9cfe339806df8847e9af4ab105c551645110b1c0576f", + "evidence/figures/upload-throughput.svg": "9fc4c2946a74f047f5f0c83291ba77f05653e3344c34e9c99f0eea7d994deeed", + "evidence/final-check-test.txt": "cc1c058c861dc83a1202ed2b81675a4042a46a0c203d45f27ce5c99004aa18b3", + "evidence/hardware.json": "52bcfcbf973798f88ca4a441af5a7b0122e69a21355a802c92dcc2a5544ae912", + "evidence/level4.json": "3752d90c6e5d4e5cb7c5b8db6124873e9b87af642f3ea98a82240241937540d7", + "evidence/make-check-test.txt": "cc1c058c861dc83a1202ed2b81675a4042a46a0c203d45f27ce5c99004aa18b3", + "evidence/operations.json": "c222ef078951ad6e3a40515bd8a56188d0a0c2b1c1de8ba92f9f98976a235216", + "evidence/operations.txt": "b6bfd6e303838f66a333904a6fae16920a66e13cad90709e595859d79c0ab78a", + "evidence/persistence.txt": "e13f00023e004d83b2dfa575242fd3823985ac04e9e4107fb714acf0ccee07bd", + "evidence/reconcile.json": "a2150e0ed2951ba01e75c00c7257c8e285875aa58b3145354beab5a634f69b1a", + "evidence/repro-build.txt": "0300c2aa5019aaaa611960d82580bdfd8f53c540c494c00a2f40b73ce2b895bd", + "evidence/runtime.json": "d810634b8aed4c25c52ef9a0ad1a8fa7d73f76d1083b10a52ac260149d772616", + "evidence/s3-source-hashes.json": "9d0d1f414944cd6f0662067b9b2aa1c7fab7755baaa7b5560cd8740666e10f64", + "evidence/smoke.txt": "85fe65743939225bcec55093d4338aefd64679b5e457f5a6c8c07adff32d0743", + "evidence/sources-manifest.json": "44e11ad93dd156b2ebdfd99e91f2766400ffe4c5573701bebe37ee7b69b0a785", + "evidence/summary-output.txt": "2f7e52f1ac2a00d5999813149db612ba51caab3d043330974b598e1ed1528e09", + "evidence/summary.json": "4182e194ed6d12effd557bd5263b9f5a7fecfc5c6c8510b6214c183b4f808d2d", + "evidence/transfer.md": "3f874f17971915ba929aa3c31fdc79762d98baebfdf387cfcd327e78bb81d852", + "evidence/upload-load-output.txt": "bdeaae801cf295bb97b87ac3b8ef7878a7a0679d36bb43d5a1dc49d84b5da6d7", + "evidence/upload-load.json": "d965adbcfb4313e8529537129a65126ca17c8511758c91dd792b3b249b681e7d", + "evidence/upload-summary.csv": "9426de7d7c84be13bb85ef528fcea0da6cda9980e16445b19920f9129b768679", + "evidence/validate.txt": "69f17e4bbfdcc30ca307ad250746fef7007e94193586ea01e13e4040e1048e97", + "examples/requests.http": "53eea65721ef1c3e18470dbb95c894611c08856bab650fa0cfc0721f042cae91", + "infra/README.md": "33a4f7b870fd0b2bed98d46daf26d45dd7436176739672925246f8e3fe7c2278", + "infra/bucket.sh": "f7aed7c75255555c1b773adf2686d3206b113e5f0af863b96c833a63470370f6", + "infra/haproxy.cfg": "b359e64c9c1b923b85826db8c111a59fce4e26f788a784662b9a5046e65c226b", + "infra/reconciliation.md": "a3855637ffbdbfa06ef5f03744a068b9088a51e025f373330467f12a9aa27303", + "infra/s3-entrypoint.sh": "9b8220e117638e6a16c75cc6c824718209e7d5a767ed0b28f7038b58d676ec8c", + "infra/s3-policy.json": "463b042e97c4982be9b44ca22708c0d758d3f764c1cc7bb49710356a8549821b", + "lab.json": "30fd674b572fb0bf3a6845ce62987305db7f5b06d87f20aae72a1e30e566b185", + "lab4.env.example": "8c6da0b5707bb7e00c94fb3f626378412d335c0d6af2e29323e2e88323dc918c", + "migrations/001_initial.sql": "d4c044482ba61b6a59b35cb02cff16b650fa0fb813242c0678d45a3564e86340", + "migrations/002_idempotency.sql": "b3938061613446d1676dfdf41af1b8417dd0396d2a65e4d3d6b263d311ccdbce", + "migrations/003_attachments.sql": "4137877de095ca8cb590582bfa59758c31248f013e9b99ddcf6a2e2bedaff125", + "requirements-dev.txt": "e70f5e9db859ef3439dc6ae647890a18cd7a2752889598bbd19c822f31b7c499", + "requirements-research.txt": "1ff1ed1bc02424e20028427bd35d1b272c9700126033add35adf4566650dc067", + "requirements.lock.txt": "580f40ed0e8de26de95ac957e5b048a81d586c6d9861df5dfecd99700f57b072", + "requirements.txt": "dc14f80eca7ded138ec7fff542192d8013b74c458c3dd3234b11b6a391f75bf0", + "scripts/actions.py": "c58c1c5a72d474e38907440e0dea9914196bb0697cd1235c4b36e6335b151fdd", + "scripts/audit_solution.py": "609c740570364bec39f7d26bf85176537aea8e007fb21af5e0bedd12e6c60942", + "scripts/bootstrap.py": "db88a2aeba07281cd5ae8a001f0455bf8b5e19639bb06462513687fc59717aca", + "scripts/check_contract.py": "61e300fe5c1074360418e24b4653b12f96808247a2beaf2a64859aa360f81493", + "scripts/plot_results.py": "fecbeb33d73e7d3cbdeda41701f3be40dd6d04f974df1e85bd5c9a2152182c64", + "scripts/restart.cmd": "e008c8fa1ce5cacf582e502179ec78e15d45fe90dda1f4c929b7a01052fe3b11", + "scripts/restart.example.sh": "0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b", + "scripts/restart.sh": "fcea33ea66beeb9b2ca6964a00c817a45afb7626eb9e4deb92d03adfc78a70b2", + "scripts/run_upload_load.py": "d1959fe3756fecbaa1923c972dcd41678634100b5f59a10bcab97851cb9d25cf", + "scripts/stop-one.cmd": "95e787774e6ffb007dc53c94a24c91d4a6ff3c2ae2bb15198cf7d4ccb06664ea", + "scripts/stop-one.example.sh": "1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811", + "scripts/stop-one.sh": "82a0c50fd834c9ce330ec4b6b808c64c7d76dda7352ae4210a3322c9e0f2e125", + "scripts/validate_openapi.py": "cd7a33221b1860c5d2330e52afa6cb222d375db671ae9ab379ba51a0d3a77946", + "scripts/verify_clean_export.py": "a28a9388c416e101792b7587a784d127b2e7c78dd56da0bf9fa40d384533c52c", + "scripts/wait_ready.py": "29e9965053ad2cba45dc68a4cc462a7ccd2c6cbd9da9ed89351721f926f5643c", + "src/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "src/app.py": "0cb5d0151e73166aed98acfe151415a93f18221cf27ebd8e39f433cda30b8f13", + "src/cache.py": "bfc8f1aa90bee4625b3887a3487be085560b7a2705cc94dcb557dd6c03062028", + "src/config.py": "aafa2db508658d2d5467d2c57ed5f4dea11609ab5928f86abc1e35fc8ba923d7", + "src/main.py": "3f49c442758f1a4c3a156a4555f2b3f6d8dff0529fc62cb763f11eaaa2d2760b", + "src/migrate.py": "48c2819b34e22ee099a97d8742a65c87bc0abf9cd617cf6b6caae8a1ad145af5", + "src/models.py": "afc5b651059a440904e1c4ac509ac8e24796bceac5869a6421e42d388f2fe6d6", + "src/objects.py": "75f15c13be0b5eecf441a437a8250a0d6367b7b098b288a2cd1994c0b1a4321a", + "src/reconcile.py": "228a7994ce5b4110d7391734b0885c01ece754912c285c7f530427ec2674f133", + "src/routes/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "src/routes/attachments.py": "5518b8a264b8b0454e30606123119e83382159f2f031864e33a3e48a62496c84", + "src/routes/chats.py": "46ef834fce05037c2ecab26192ed58c28be81afc6a1db1375560f11bb9d63c12", + "src/routes/common.py": "fa2df76c819abe9a99b8b380a50b2aa4556aa43ac84afa95cac574aab8468a7e", + "src/routes/health.py": "a2d7aef51d4fff4f4c8523a4b1d1b894747f945ac3dd4d26b521d6a228cb88be", + "src/routes/messages.py": "7dc016d1252f07538c06b7c155d3efb8af02ac2a2a9867267168a74e7cf0bd3c", + "src/routes/users.py": "f5d4c7e4c4e5b1abfde7869a46e8a81a935b32f4809ddb8b2a4cb13175a24f50", + "src/storage.py": "81f5bcb0bc6ce356c76f1ea0b90d74566d436ca6f2dcc35e80cc0ae512588979", + "student-tests/README.md": "eb43baa6868c409f71905c7ab11b0bb187a8e9a87fc4842909949b270107e354", + "student-tests/test_attachments.py": "e26fece8adc6a63bcccfe6c1de53e3899f0c43d0c8c9fd6502c988403a9a95e5", + "student-tests/test_operations.py": "7a62d819afdaea80767cb2ff4e8141eebce9e564ec20241c7d3f5b645d742999", + "student-tests/test_reconcile.py": "f1e0f7461f57fa0aad2a16e118fe86ed5852ec1d86caac07ea81a829c7f2f787", + "student-tests/upload_load.py": "022eea2772fcbf780042b3959c516788929dfde4968dcd9f7caa2531999c52e9", + "tests/README.md": "89afb3063ec0ccd7cb68fdac1de57513f9cc81b77bad96f202daaca2b66a5e82", + "tests/client.py": "b318084e2a4c380db91a55d082190bf33979fa556a4c4ca87e01f46ae275775c", + "tests/fixtures/README.md": "0eaa4e8837032d1786fc0c4b1576656ff3f4d1931fdd51b80e5c053f88ab3872", + "tests/fixtures/sample.png": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "tests/image_probe.py": "ef48d8882ab138b9699a173e7419e5814c22a10b01b6b0ddbb50ea49481a5baa", + "tests/load.py": "9a9aff05e109630e243c3b092d7bd564344e4872b29e38b6618647a08460a72e", + "tests/resilience.py": "4ba30b9c81a0664127ddab2b0bea5e76aa2087be8bb25aa13cd5a8e69be23fdd", + "tests/smoke.py": "354e168e1e644145c3ff50bbbe189fd217cce5659aa125f588098025ffab0fdf" + } +} \ No newline at end of file diff --git a/evidence/load-messages-resources.json b/evidence/load-messages-resources.json new file mode 100644 index 00000000..8d62c6b8 --- /dev/null +++ b/evidence/load-messages-resources.json @@ -0,0 +1,775 @@ +{ + "command": [ + "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-X", + "utf8", + "tests/load.py", + "--base-url", + "http://127.0.0.1:8085", + "--duration", + "30", + "--concurrency", + "4", + "--output", + "evidence/load-messages.json" + ], + "exit_code": 0, + "samples": [ + { + "at_seconds": 3.241, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "34.76%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.93%", + "MemUsage": "108.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "182MB / 182MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "48.21%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "19.08%", + "MemUsage": "122.1MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "137MB / 137MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "1.63%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.71MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "288kB / 253kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "33.80%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "10.18%", + "MemUsage": "52.11MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "6.3MB / 8.09MB", + "PIDs": "18" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.77%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.56MB / 1.1MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.37%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.75%", + "MemUsage": "13.76MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "317MB / 317MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.62%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "0.84%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "479kB / 521kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.03%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.03%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 8.631, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "28.24%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.97%", + "MemUsage": "108.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "182MB / 183MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "50.33%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "19.13%", + "MemUsage": "122.5MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "137MB / 137MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "53.78%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.38%", + "MemUsage": "78.73MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "288kB / 253kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "24.36%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "10.65%", + "MemUsage": "54.51MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "7.27MB / 8.97MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.88%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.58MB / 1.11MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.03%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "11.15%", + "MemUsage": "14.27MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "318MB / 318MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.65%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "0.74%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "479kB / 521kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.04%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.05%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 13.955, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "46.72%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.99%", + "MemUsage": "108.8MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "183MB / 183MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "48.40%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "19.24%", + "MemUsage": "123.1MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "138MB / 138MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "42.48%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "24.05%", + "MemUsage": "123.1MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "288kB / 253kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "30.00%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "11.38%", + "MemUsage": "58.29MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "8.2MB / 9.81MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.88%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.61MB / 1.13MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.29%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "11.87%", + "MemUsage": "15.2MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "318MB / 319MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.46%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "0.66%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "479kB / 521kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.03%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.04%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 19.25, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "39.37%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "17.00%", + "MemUsage": "108.8MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "184MB / 184MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "49.09%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "19.17%", + "MemUsage": "122.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "138MB / 139MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "0.23%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.71MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "288kB / 253kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 1.57MB", + "CPUPerc": "21.35%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "11.24%", + "MemUsage": "57.52MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "9.04MB / 10.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.67%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.63MB / 1.14MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "16.23%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "11.01%", + "MemUsage": "14.09MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "319MB / 319MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.49%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "0.79%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385.1MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "479kB / 521kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.04%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.05%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 24.605, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "49.61%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "17.00%", + "MemUsage": "108.8MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "184MB / 185MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "31.04%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "19.17%", + "MemUsage": "122.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "139MB / 140MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "75.54%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.73%", + "MemUsage": "162.4MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "288kB / 253kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 1.57MB", + "CPUPerc": "32.34%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "11.36%", + "MemUsage": "58.14MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "10.5MB / 11.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.94%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.65MB / 1.16MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.43%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "11.71%", + "MemUsage": "14.98MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "320MB / 320MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.66%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "0.64%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "479kB / 521kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.03%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.04%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 29.874, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "36.39%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "17.01%", + "MemUsage": "108.8MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "185MB / 186MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "48.11%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "19.17%", + "MemUsage": "122.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "140MB / 140MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "4.78%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "19.12%", + "MemUsage": "97.92MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "288kB / 254kB", + "PIDs": "60" + }, + { + "BlockIO": "0B / 20.3MB", + "CPUPerc": "27.03%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "11.51%", + "MemUsage": "58.93MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "11.6MB / 12.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.47%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.68MB / 1.17MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.59%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "11.22%", + "MemUsage": "14.36MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "321MB / 321MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.47%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "0.59%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.98%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "480kB / 522kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.03%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.05%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/load-messages-run.txt b/evidence/load-messages-run.txt new file mode 100644 index 00000000..2e0439d9 --- /dev/null +++ b/evidence/load-messages-run.txt @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.112, + "concurrency": 4, + "successful_requests": 2507, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 83.256, + "successful_latency_ms": { + "p50": 25.341, + "p95": 146.956, + "p99": 251.292 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-messages.json b/evidence/load-messages.json new file mode 100644 index 00000000..2e0439d9 --- /dev/null +++ b/evidence/load-messages.json @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.112, + "concurrency": 4, + "successful_requests": 2507, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 83.256, + "successful_latency_ms": { + "p50": 25.341, + "p95": 146.956, + "p99": 251.292 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-pipeline.json b/evidence/load-pipeline.json new file mode 100644 index 00000000..ea87a1ae --- /dev/null +++ b/evidence/load-pipeline.json @@ -0,0 +1,3520 @@ +[ + { + "profile": "320x200", + "bytes_per_image": 111817, + "repeat": 1, + "concurrency": 2, + "requested_duration": 20, + "elapsed_seconds": 20.093, + "warmup_completed": 1, + "completed": 80, + "successful_rps": 3.982, + "failures": [], + "error_fraction": 0.0, + "p50": 0.401, + "p95": 0.7265, + "p99": 0.7436, + "upload_p95": 0.0781, + "stats": [ + { + "at_seconds": 3.186, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "6.10%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.58%", + "MemUsage": "99.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "1.31MB / 1.41MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "16.94%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.44%", + "MemUsage": "98.81MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "505kB / 509kB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 106kB", + "CPUPerc": "0.57%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.35%", + "MemUsage": "78.61MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "22.4kB / 15.1kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "6.94%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.24%", + "MemUsage": "37.06MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "2.23MB / 2.32MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.05%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "480kB / 304kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.12%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.15%", + "MemUsage": "12.99MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "3.03MB / 3MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.64%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 2.79MB", + "CPUPerc": "5.22%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "44.74%", + "MemUsage": "343.6MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "2.98MB / 1.69MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.51%", + "MemUsage": "57.63MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "13kB / 9.44kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.33%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.05%", + "MemUsage": "61.84MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "621kB / 676kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.17%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "7.82%", + "MemUsage": "60.04MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "621kB / 676kB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 8.271, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.26%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.75%", + "MemUsage": "100.8MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "2.44MB / 2.54MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.36%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.72%", + "MemUsage": "100.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "2MB / 1.89MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 106kB", + "CPUPerc": "75.91%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.69%", + "MemUsage": "162.3MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "38.2kB / 29kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "5.28%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.43%", + "MemUsage": "38.06MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "2.44MB / 2.62MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.10%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "533kB / 344kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.37%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.30%", + "MemUsage": "13.18MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "5.29MB / 5.26MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.36%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 7.52MB", + "CPUPerc": "12.12%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "45.26%", + "MemUsage": "347.6MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "7.68MB / 4.1MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.96%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.52%", + "MemUsage": "57.64MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "50.7kB / 50.9kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "23.37%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.17%", + "MemUsage": "62.73MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "1.84MB / 2.01MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "28.57%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "7.93%", + "MemUsage": "60.92MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "1.84MB / 2.01MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 13.371, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "6.80%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.87%", + "MemUsage": "101.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "3.57MB / 3.67MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.67%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.71%", + "MemUsage": "100.5MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "3.36MB / 3.38MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 106kB", + "CPUPerc": "1.38%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.36%", + "MemUsage": "78.62MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "54.3kB / 43.5kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "7.67%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.79%", + "MemUsage": "39.87MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "2.64MB / 2.91MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.24%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "586kB / 385kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.73%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.34%", + "MemUsage": "13.23MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "7.67MB / 7.64MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.47%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 12.6MB", + "CPUPerc": "8.82%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "45.38%", + "MemUsage": "348.5MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "12.7MB / 6.61MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.52%", + "MemUsage": "57.64MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "50.7kB / 50.9kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "27.77%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.20%", + "MemUsage": "63.01MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "3.18MB / 3.35MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "26.44%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "7.96%", + "MemUsage": "61.1MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "3.07MB / 3.34MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 18.42, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "5.51%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.88%", + "MemUsage": "101.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "4.59MB / 4.69MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "5.27%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.87%", + "MemUsage": "101.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "4.95MB / 4.98MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "1.08%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.36%", + "MemUsage": "78.62MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "70.9kB / 58kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "3.68%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.87%", + "MemUsage": "40.29MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "2.83MB / 3.21MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.23%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "640kB / 426kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.97%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.84%", + "MemUsage": "13.87MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "10.3MB / 10.2MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.44%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 17.8MB", + "CPUPerc": "7.51%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "41.96%", + "MemUsage": "322.3MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "18MB / 9.13MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.52%", + "MemUsage": "57.64MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "50.7kB / 50.9kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "30.74%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.31%", + "MemUsage": "63.84MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "4.41MB / 4.81MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "31.68%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.08%", + "MemUsage": "62.02MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "4.41MB / 4.81MB", + "PIDs": "5" + } + ] + } + ] + }, + { + "profile": "320x200", + "bytes_per_image": 111817, + "repeat": 2, + "concurrency": 2, + "requested_duration": 20, + "elapsed_seconds": 20.24, + "warmup_completed": 1, + "completed": 73, + "successful_rps": 3.607, + "failures": [], + "error_fraction": 0.0, + "p50": 0.4219, + "p95": 0.7366, + "p99": 0.7857, + "upload_p95": 0.0893, + "stats": [ + { + "at_seconds": 2.572, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "18.46%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.88%", + "MemUsage": "101.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "5.94MB / 6.04MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.80%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.98%", + "MemUsage": "102.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "5.96MB / 5.99MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "75.65%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.96%", + "MemUsage": "163.7MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "84.9kB / 70.6kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "4.89%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.98%", + "MemUsage": "40.88MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3MB / 3.47MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.67%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.70%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "688kB / 462kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.84%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.37%", + "MemUsage": "13.27MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "12.4MB / 12.4MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.50%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.80%", + "MemUsage": "3.59MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 22.4MB", + "CPUPerc": "8.34%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "42.04%", + "MemUsage": "322.9MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "22.5MB / 11.3MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.52%", + "MemUsage": "57.64MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "50.7kB / 50.9kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "16.56%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.42%", + "MemUsage": "64.67MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "5.52MB / 6.01MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "32.08%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.18%", + "MemUsage": "62.84MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "5.52MB / 6.01MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 7.579, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.71%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.90%", + "MemUsage": "101.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "7.17MB / 7.28MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.33%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.98%", + "MemUsage": "102.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "6.74MB / 6.76MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "1.11%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.36%", + "MemUsage": "78.63MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "97.9kB / 82.3kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "7.51%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.94%", + "MemUsage": "40.66MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3.16MB / 3.71MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.89%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "734kB / 497kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.68%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.57%", + "MemUsage": "13.53MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "14.3MB / 14.3MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.47%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 26.3MB", + "CPUPerc": "5.38%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "42.10%", + "MemUsage": "323.3MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "26.4MB / 13.3MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.52%", + "MemUsage": "57.64MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "50.7kB / 50.9kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "21.67%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "9.74%", + "MemUsage": "74.78MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "6.61MB / 7.08MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "21.32%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.24%", + "MemUsage": "63.3MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "6.49MB / 6.95MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 12.595, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.93%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.90%", + "MemUsage": "101.8MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "8.19MB / 8.29MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.27%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "15.99%", + "MemUsage": "102.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "8.09MB / 8.11MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "0.91%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.36%", + "MemUsage": "78.64MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "113kB / 95.5kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "3.37%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.91%", + "MemUsage": "40.48MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3.34MB / 3.98MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.35%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "784kB / 535kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.41%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.40%", + "MemUsage": "13.31MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "16.6MB / 16.6MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.47%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 30.9MB", + "CPUPerc": "6.08%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "42.27%", + "MemUsage": "324.6MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "31MB / 15.6MB", + "PIDs": "21" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.52%", + "MemUsage": "57.64MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "50.7kB / 50.9kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 262kB", + "CPUPerc": "19.44%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.50%", + "MemUsage": "65.27MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "7.71MB / 8.28MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "32.45%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.24%", + "MemUsage": "63.27MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "7.71MB / 8.28MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 17.542, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "8.61%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "15.99%", + "MemUsage": "102.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "9.31MB / 9.41MB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "8.94%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.00%", + "MemUsage": "102.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "9.33MB / 9.35MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "75.17%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.62%", + "MemUsage": "161.9MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "127kB / 108kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "15.41%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.92%", + "MemUsage": "40.55MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3.52MB / 4.26MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.32%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "831kB / 571kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.60%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.42%", + "MemUsage": "13.33MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "18.9MB / 18.8MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.50%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 35.7MB", + "CPUPerc": "16.20%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "38.31%", + "MemUsage": "294.2MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "35.8MB / 17.8MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.80%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.62%", + "MemUsage": "57.9MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "105kB / 85.3kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 262kB", + "CPUPerc": "26.49%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.45%", + "MemUsage": "64.89MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "8.82MB / 9.61MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "28.54%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.21%", + "MemUsage": "63.05MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "8.82MB / 9.61MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 22.662, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.38%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.03%", + "MemUsage": "102.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "9.95MB / 10.1MB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.20%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.00%", + "MemUsage": "102.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "10.1MB / 10.1MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "0.27%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.41%", + "MemUsage": "78.89MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "136kB / 116kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "0.37%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.97%", + "MemUsage": "40.79MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3.7MB / 4.49MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.43%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "872kB / 601kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.25%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.39%", + "MemUsage": "13.29MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "20.2MB / 20.2MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.53%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 38.5MB", + "CPUPerc": "0.53%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "39.63%", + "MemUsage": "304.3MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "38.7MB / 19.3MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.63%", + "MemUsage": "57.95MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "205kB / 220kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 262kB", + "CPUPerc": "0.04%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.45%", + "MemUsage": "64.9MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "9.55MB / 10.4MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.04%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.21%", + "MemUsage": "63.07MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "9.55MB / 10.4MB", + "PIDs": "5" + } + ] + } + ] + }, + { + "profile": "320x200", + "bytes_per_image": 111817, + "repeat": 3, + "concurrency": 2, + "requested_duration": 20, + "elapsed_seconds": 20.004, + "warmup_completed": 1, + "completed": 71, + "successful_rps": 3.549, + "failures": [], + "error_fraction": 0.0, + "p50": 0.4389, + "p95": 0.7694, + "p99": 0.82, + "upload_p95": 0.089, + "stats": [ + { + "at_seconds": 3.167, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.89%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.02%", + "MemUsage": "102.5MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "10.6MB / 10.7MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.55%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.01%", + "MemUsage": "102.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "10.6MB / 10.6MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "1.08%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.68MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "143kB / 123kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "4.92%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.97%", + "MemUsage": "40.79MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3.79MB / 4.63MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.90%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "902kB / 623kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.74%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.83%", + "MemUsage": "13.86MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "21MB / 21MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.44%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.80%", + "MemUsage": "3.59MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 40.6MB", + "CPUPerc": "4.68%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "39.66%", + "MemUsage": "304.6MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "40.8MB / 20.4MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.63%", + "MemUsage": "57.95MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "205kB / 220kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 262kB", + "CPUPerc": "18.74%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.50%", + "MemUsage": "65.31MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "10.2MB / 10.9MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "20.28%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.21%", + "MemUsage": "63.08MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "10MB / 10.9MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 8.313, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.47%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.02%", + "MemUsage": "102.5MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "11.7MB / 11.8MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.72%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.83%", + "MemUsage": "107.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "11.8MB / 11.9MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "72.76%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.15%", + "MemUsage": "159.5MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "157kB / 135kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "3.24%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "7.98%", + "MemUsage": "40.88MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "3.97MB / 4.9MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.37%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "944kB / 655kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.82%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.48%", + "MemUsage": "13.41MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "23.5MB / 23.5MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.46%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 45.2MB", + "CPUPerc": "12.60%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "39.75%", + "MemUsage": "305.3MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "45.5MB / 22.5MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.63%", + "MemUsage": "57.95MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "205kB / 220kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 262kB", + "CPUPerc": "19.14%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.46%", + "MemUsage": "64.94MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "11.3MB / 12.3MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.39%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.22%", + "MemUsage": "63.09MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "11.1MB / 12.1MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 13.541, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "22.94%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.04%", + "MemUsage": "102.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "12.7MB / 12.8MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.37%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.00%", + "MemUsage": "102.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "12.6MB / 12.6MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "1.14%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.68MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "172kB / 149kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 778kB", + "CPUPerc": "7.65%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.07%", + "MemUsage": "41.32MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.14MB / 5.16MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.14%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1MB / 698kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.15%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.43%", + "MemUsage": "13.36MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "25.7MB / 25.6MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.44%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.80%", + "MemUsage": "3.59MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 49.7MB", + "CPUPerc": "7.57%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "39.82%", + "MemUsage": "305.8MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "49.9MB / 24.8MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.63%", + "MemUsage": "57.95MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "205kB / 220kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 262kB", + "CPUPerc": "33.11%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.46%", + "MemUsage": "64.96MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "12.4MB / 13.5MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "23.90%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "9.16%", + "MemUsage": "70.37MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "12.4MB / 13.3MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 18.863, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.62%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.03%", + "MemUsage": "102.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "13.9MB / 14.1MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "6.32%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.00%", + "MemUsage": "102.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "14.2MB / 14.2MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 143kB", + "CPUPerc": "0.92%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.68MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "189kB / 164kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "4.02%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.02%", + "MemUsage": "41.04MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.34MB / 5.46MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.10%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.06MB / 741kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.78%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.47%", + "MemUsage": "13.4MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "28.3MB / 28.3MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.48%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.80%", + "MemUsage": "3.59MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 54.9MB", + "CPUPerc": "7.08%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "39.94%", + "MemUsage": "306.8MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "55.1MB / 27.3MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.63%", + "MemUsage": "57.95MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "205kB / 220kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 389kB", + "CPUPerc": "24.86%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.80%", + "MemUsage": "67.59MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "13.7MB / 14.8MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "33.49%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.22%", + "MemUsage": "63.16MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "13.3MB / 14.5MB", + "PIDs": "5" + } + ] + } + ] + }, + { + "profile": "1024x768", + "bytes_per_image": 2360847, + "repeat": 1, + "concurrency": 2, + "requested_duration": 20, + "elapsed_seconds": 20.567, + "warmup_completed": 1, + "completed": 45, + "successful_rps": 2.188, + "failures": [], + "error_fraction": 0.0, + "p50": 0.8308, + "p95": 1.1487, + "p99": 1.1615, + "upload_p95": 0.2497, + "stats": [ + { + "at_seconds": 3.238, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "17.26%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.74%", + "MemUsage": "107.1MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "21.7MB / 21.8MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.95%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.71%", + "MemUsage": "106.9MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "24MB / 24.1MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 143kB", + "CPUPerc": "73.57%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.16%", + "MemUsage": "159.5MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "197kB / 172kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "6.43%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.05%", + "MemUsage": "41.2MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.46MB / 5.65MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.06%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.1MB / 769kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.77%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.48%", + "MemUsage": "13.41MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "45.4MB / 45.4MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.54%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.80%", + "MemUsage": "3.59MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 87.6MB", + "CPUPerc": "15.62%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "40.12%", + "MemUsage": "308.1MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "87.8MB / 44.5MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.63%", + "MemUsage": "57.95MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "205kB / 220kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 389kB", + "CPUPerc": "31.88%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "10.69%", + "MemUsage": "82.09MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "23.5MB / 24.1MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "31.23%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.27%", + "MemUsage": "63.51MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "18.7MB / 18.2MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 8.467, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.21%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.76%", + "MemUsage": "107.2MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "38.4MB / 38.5MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.69%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.72%", + "MemUsage": "107MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "36MB / 36MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 143kB", + "CPUPerc": "0.61%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.68MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "207kB / 180kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "2.56%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.07%", + "MemUsage": "41.29MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.65MB / 5.9MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.85%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.14MB / 798kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.74%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.69%", + "MemUsage": "13.68MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "74MB / 74MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.38%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 152MB", + "CPUPerc": "15.77%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "42.55%", + "MemUsage": "326.8MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "152MB / 73.1MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "329kB / 355kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 389kB", + "CPUPerc": "38.39%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "11.89%", + "MemUsage": "91.32MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "37.7MB / 41.9MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 127kB", + "CPUPerc": "23.82%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "11.46%", + "MemUsage": "87.98MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "35.4MB / 38.9MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 13.441, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "19.80%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "17.20%", + "MemUsage": "110.1MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "62.2MB / 62.3MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "6.72%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.72%", + "MemUsage": "107MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "38.4MB / 38.5MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "20.44%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.68MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "215kB / 187kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "6.24%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.17%", + "MemUsage": "41.83MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.77MB / 6.08MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.71%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.17MB / 825kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.83%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.50%", + "MemUsage": "13.44MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "100MB / 100MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.57%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 210MB", + "CPUPerc": "13.70%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "42.87%", + "MemUsage": "329.2MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "211MB / 96.9MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "329kB / 355kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 389kB", + "CPUPerc": "33.90%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "13.01%", + "MemUsage": "99.91MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "49.6MB / 56.7MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 2.52MB", + "CPUPerc": "27.32%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.25%", + "MemUsage": "63.33MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "47.3MB / 56.8MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 18.47, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "19.08%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.76%", + "MemUsage": "107.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "76.6MB / 76.7MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.38%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.72%", + "MemUsage": "107MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "52.7MB / 52.8MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "56.27%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "29.65%", + "MemUsage": "151.8MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "224kB / 196kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "3.83%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.13%", + "MemUsage": "41.61MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.91MB / 6.27MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.42%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.70%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.21MB / 854kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.01%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.50%", + "MemUsage": "13.44MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "129MB / 129MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.50%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 274MB", + "CPUPerc": "10.91%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "43.17%", + "MemUsage": "331.6MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "274MB / 125MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "329kB / 355kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 389kB", + "CPUPerc": "50.93%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "11.50%", + "MemUsage": "88.29MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "63.9MB / 74.4MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 2.52MB", + "CPUPerc": "37.36%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.26%", + "MemUsage": "63.46MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "61.5MB / 74.6MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 23.445, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.57%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.76%", + "MemUsage": "107.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "81.4MB / 81.5MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.59%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.72%", + "MemUsage": "107MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "57.5MB / 57.6MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "0.24%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.69MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "228kB / 199kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "2.43%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.15%", + "MemUsage": "41.73MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "4.98MB / 6.38MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.36%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.24MB / 873kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.28%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.47%", + "MemUsage": "13.41MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "138MB / 138MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.55%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 301MB", + "CPUPerc": "0.50%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "43.86%", + "MemUsage": "336.9MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "302MB / 137MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "329kB / 355kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 389kB", + "CPUPerc": "0.03%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.53%", + "MemUsage": "65.5MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "68.6MB / 83.3MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 2.52MB", + "CPUPerc": "0.04%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.28%", + "MemUsage": "63.62MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "68.6MB / 83.5MB", + "PIDs": "5" + } + ] + } + ] + }, + { + "profile": "1024x768", + "bytes_per_image": 2360847, + "repeat": 2, + "concurrency": 2, + "requested_duration": 20, + "elapsed_seconds": 20.396, + "warmup_completed": 1, + "completed": 32, + "successful_rps": 1.569, + "failures": [ + "Failure: GET /api/v1/attachments/63d3337b-5ac0-478b-9e30-cde11dd2fd07: connection/TLS failure (URLError)" + ], + "error_fraction": 0.030303030303030304, + "p50": 0.79, + "p95": 1.1397, + "p99": 1.2381, + "upload_p95": 0.1837, + "stats": [ + { + "at_seconds": 3.161, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.23%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.76%", + "MemUsage": "107.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "88.5MB / 88.6MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "5.61%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.72%", + "MemUsage": "107MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "62.3MB / 62.4MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "75.36%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.14%", + "MemUsage": "159.5MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "232kB / 203kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "1.58%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.08%", + "MemUsage": "41.39MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.03MB / 6.46MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.77%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.27MB / 890kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.48%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.50%", + "MemUsage": "13.44MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "150MB / 150MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.51%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 325MB", + "CPUPerc": "5.04%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "44.82%", + "MemUsage": "344.2MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "325MB / 149MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "329kB / 355kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 2.75MB", + "CPUPerc": "13.48%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "10.71%", + "MemUsage": "82.23MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "75.7MB / 89.2MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 4.91MB", + "CPUPerc": "32.77%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.30%", + "MemUsage": "63.75MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "73.4MB / 89.4MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 8.318, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.20%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.78%", + "MemUsage": "107.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "98.1MB / 98.2MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.72%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "18.13%", + "MemUsage": "116MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "64.7MB / 64.8MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "1.08%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.69MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "237kB / 207kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "5.04%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.14%", + "MemUsage": "41.66MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.12MB / 6.58MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.54%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.3MB / 911kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.32%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.49%", + "MemUsage": "13.43MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "165MB / 165MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.37%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 360MB", + "CPUPerc": "6.87%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "44.98%", + "MemUsage": "345.4MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "360MB / 163MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "329kB / 355kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 2.75MB", + "CPUPerc": "23.99%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.55%", + "MemUsage": "65.7MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "82.9MB / 101MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 4.91MB", + "CPUPerc": "18.08%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.31%", + "MemUsage": "63.82MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "80.5MB / 98.3MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 13.639, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.42%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.83%", + "MemUsage": "107.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "115MB / 115MB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "18.62%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.72%", + "MemUsage": "107MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "71.9MB / 72MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "0.55%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.7MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "243kB / 213kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 786kB", + "CPUPerc": "9.98%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.14%", + "MemUsage": "41.68MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.22MB / 6.72MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.83%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.33MB / 937kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.29%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.53%", + "MemUsage": "13.48MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "186MB / 186MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.48%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 402MB", + "CPUPerc": "11.15%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "46.76%", + "MemUsage": "359.1MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "402MB / 185MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.36%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "400kB / 435kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 2.75MB", + "CPUPerc": "46.72%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "13.07%", + "MemUsage": "100.4MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "92.4MB / 110MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 4.91MB", + "CPUPerc": "29.55%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "9.31%", + "MemUsage": "71.53MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "90MB / 107MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 18.881, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.00%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "17.06%", + "MemUsage": "109.2MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "127MB / 127MB", + "PIDs": "18" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "24.09%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "17.08%", + "MemUsage": "109.3MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "83.8MB / 81.6MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "76.02%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "31.11%", + "MemUsage": "159.3MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "252kB / 221kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 811kB", + "CPUPerc": "4.71%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.15%", + "MemUsage": "41.73MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.38MB / 6.95MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.08%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.37MB / 964kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.26%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.53%", + "MemUsage": "13.47MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "210MB / 210MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.51%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 461MB", + "CPUPerc": "19.87%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "49.59%", + "MemUsage": "380.9MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "462MB / 209MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "400kB / 435kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "39.48%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.56%", + "MemUsage": "65.73MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "104MB / 128MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 7.87MB", + "CPUPerc": "37.49%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.88MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "104MB / 128MB", + "PIDs": "5" + } + ] + } + ] + }, + { + "profile": "1024x768", + "bytes_per_image": 2360847, + "repeat": 3, + "concurrency": 2, + "requested_duration": 20, + "elapsed_seconds": 20.969, + "warmup_completed": 1, + "completed": 40, + "successful_rps": 1.908, + "failures": [], + "error_fraction": 0.0, + "p50": 1.0675, + "p95": 1.2299, + "p99": 1.3224, + "upload_p95": 0.2471, + "stats": [ + { + "at_seconds": 3.248, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "24.77%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.82%", + "MemUsage": "107.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "143MB / 144MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.43%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "17.64%", + "MemUsage": "112.9MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "93.4MB / 93.6MB", + "PIDs": "13" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "0.85%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.7MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "260kB / 228kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 811kB", + "CPUPerc": "6.77%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.14%", + "MemUsage": "41.68MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.5MB / 7.13MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.14%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.41MB / 994kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.60%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.56%", + "MemUsage": "13.51MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "236MB / 236MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.56%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 514MB", + "CPUPerc": "11.92%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "49.65%", + "MemUsage": "381.3MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "514MB / 235MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "400kB / 435kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "39.45%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "9.07%", + "MemUsage": "69.68MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "119MB / 142MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 7.87MB", + "CPUPerc": "53.17%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "11.88%", + "MemUsage": "91.25MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "116MB / 140MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 8.179, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.43%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.82%", + "MemUsage": "107.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "153MB / 153MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "19.29%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.78%", + "MemUsage": "107.4MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "108MB / 108MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "1.60%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.7MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "268kB / 235kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "6.97%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.15%", + "MemUsage": "41.7MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.62MB / 7.3MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.42%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.44MB / 1.02MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.22%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.96%", + "MemUsage": "14.03MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "260MB / 260MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.50%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 567MB", + "CPUPerc": "11.16%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "49.76%", + "MemUsage": "382.2MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "568MB / 256MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "400kB / 435kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "35.85%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.56%", + "MemUsage": "65.78MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "128MB / 157MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 7.87MB", + "CPUPerc": "46.26%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "10.35%", + "MemUsage": "79.46MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "128MB / 154MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 13.271, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.22%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.82%", + "MemUsage": "107.7MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "167MB / 168MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "22.82%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.75%", + "MemUsage": "107.2MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "115MB / 115MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "77.58%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "30.99%", + "MemUsage": "158.7MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "275kB / 241kB", + "PIDs": "107" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "3.48%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.15%", + "MemUsage": "41.73MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.74MB / 7.48MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.92%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.48MB / 1.04MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.66%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.57%", + "MemUsage": "13.53MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "281MB / 281MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.57%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 618MB", + "CPUPerc": "11.88%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "49.86%", + "MemUsage": "382.9MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "618MB / 278MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "400kB / 435kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "44.75%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.79MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "140MB / 172MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "45.42%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.91MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "138MB / 169MB", + "PIDs": "5" + } + ] + }, + { + "at_seconds": 18.168, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.55%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.81%", + "MemUsage": "107.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "177MB / 177MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.46%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.75%", + "MemUsage": "107.2MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "127MB / 127MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "0.57%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.71MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "282kB / 248kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "6.22%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.19%", + "MemUsage": "41.95MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.85MB / 7.64MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.82%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.69%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.52MB / 1.07MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.72%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "11.17%", + "MemUsage": "14.3MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "303MB / 303MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.45%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 663MB", + "CPUPerc": "10.01%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.06%", + "MemUsage": "384.4MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "663MB / 301MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.64%", + "MemUsage": "57.96MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "400kB / 435kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "36.46%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "13.11%", + "MemUsage": "100.7MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "152MB / 184MB", + "PIDs": "6" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "35.49%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "12.87%", + "MemUsage": "98.85MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "149MB / 181MB", + "PIDs": "6" + } + ] + }, + { + "at_seconds": 23.234, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.76%", + "Container": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "ID": "b3f8a331a598", + "MemPerc": "16.82%", + "MemUsage": "107.6MiB / 640MiB", + "Name": "lyakhonov-lab5-api-1", + "NetIO": "182MB / 182MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.56%", + "Container": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "ID": "a0acb99b68a1", + "MemPerc": "16.75%", + "MemUsage": "107.2MiB / 640MiB", + "Name": "lyakhonov-lab5-api-2", + "NetIO": "136MB / 137MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 160kB", + "CPUPerc": "65.46%", + "Container": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "ID": "b94a25ac1bb3", + "MemPerc": "15.37%", + "MemUsage": "78.71MiB / 512MiB", + "Name": "lyakhonov-lab5-broker-1", + "NetIO": "287kB / 253kB", + "PIDs": "57" + }, + { + "BlockIO": "0B / 1.02MB", + "CPUPerc": "0.94%", + "Container": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "ID": "10408d41f3b5", + "MemPerc": "8.15%", + "MemUsage": "41.71MiB / 512MiB", + "Name": "lyakhonov-lab5-db-1", + "NetIO": "5.97MB / 7.81MB", + "PIDs": "14" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.52%", + "Container": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "ID": "bdedd08309bb", + "MemPerc": "16.70%", + "MemUsage": "42.74MiB / 256MiB", + "Name": "lyakhonov-lab5-dispatcher-1", + "NetIO": "1.55MB / 1.09MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.26%", + "Container": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "ID": "934fcb61b954", + "MemPerc": "10.56%", + "MemUsage": "13.51MiB / 128MiB", + "Name": "lyakhonov-lab5-lb-1", + "NetIO": "317MB / 317MB", + "PIDs": "17" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.45%", + "Container": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "ID": "c3962c31749b", + "MemPerc": "2.61%", + "MemUsage": "3.34MiB / 128MiB", + "Name": "lyakhonov-lab5-redis-1", + "NetIO": "13kB / 10.4kB", + "PIDs": "6" + }, + { + "BlockIO": "24.6kB / 701MB", + "CPUPerc": "1.58%", + "Container": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "ID": "0b09dce98568", + "MemPerc": "50.14%", + "MemUsage": "385MiB / 768MiB", + "Name": "lyakhonov-lab5-s3-1", + "NetIO": "701MB / 316MB", + "PIDs": "22" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.82%", + "Container": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "ID": "23b814fb71a8", + "MemPerc": "22.69%", + "MemUsage": "58.09MiB / 256MiB", + "Name": "lyakhonov-lab5-staging-cleaner-1", + "NetIO": "479kB / 521kB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 5.12MB", + "CPUPerc": "0.04%", + "Container": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "ID": "1f7242787975", + "MemPerc": "8.57%", + "MemUsage": "65.83MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-1", + "NetIO": "159MB / 196MB", + "PIDs": "5" + }, + { + "BlockIO": "0B / 10.3MB", + "CPUPerc": "0.04%", + "Container": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "ID": "e36e30c0bc23", + "MemPerc": "8.32%", + "MemUsage": "63.93MiB / 768MiB", + "Name": "lyakhonov-lab5-worker-2", + "NetIO": "157MB / 193MB", + "PIDs": "5" + } + ] + } + ] + } +] \ No newline at end of file diff --git a/evidence/load-run.txt b/evidence/load-run.txt new file mode 100644 index 00000000..abbcc961 --- /dev/null +++ b/evidence/load-run.txt @@ -0,0 +1,6 @@ +{"profile": "320x200", "bytes_per_image": 111817, "repeat": 1, "concurrency": 2, "requested_duration": 20, "elapsed_seconds": 20.093, "warmup_completed": 1, "completed": 80, "successful_rps": 3.982, "failures": [], "error_fraction": 0.0, "p50": 0.401, "p95": 0.7265, "p99": 0.7436, "upload_p95": 0.0781} +{"profile": "320x200", "bytes_per_image": 111817, "repeat": 2, "concurrency": 2, "requested_duration": 20, "elapsed_seconds": 20.24, "warmup_completed": 1, "completed": 73, "successful_rps": 3.607, "failures": [], "error_fraction": 0.0, "p50": 0.4219, "p95": 0.7366, "p99": 0.7857, "upload_p95": 0.0893} +{"profile": "320x200", "bytes_per_image": 111817, "repeat": 3, "concurrency": 2, "requested_duration": 20, "elapsed_seconds": 20.004, "warmup_completed": 1, "completed": 71, "successful_rps": 3.549, "failures": [], "error_fraction": 0.0, "p50": 0.4389, "p95": 0.7694, "p99": 0.82, "upload_p95": 0.089} +{"profile": "1024x768", "bytes_per_image": 2360847, "repeat": 1, "concurrency": 2, "requested_duration": 20, "elapsed_seconds": 20.567, "warmup_completed": 1, "completed": 45, "successful_rps": 2.188, "failures": [], "error_fraction": 0.0, "p50": 0.8308, "p95": 1.1487, "p99": 1.1615, "upload_p95": 0.2497} +{"profile": "1024x768", "bytes_per_image": 2360847, "repeat": 2, "concurrency": 2, "requested_duration": 20, "elapsed_seconds": 20.396, "warmup_completed": 1, "completed": 32, "successful_rps": 1.569, "failures": ["Failure: GET /api/v1/attachments/63d3337b-5ac0-478b-9e30-cde11dd2fd07: connection/TLS failure (URLError)"], "error_fraction": 0.030303030303030304, "p50": 0.79, "p95": 1.1397, "p99": 1.2381, "upload_p95": 0.1837} +{"profile": "1024x768", "bytes_per_image": 2360847, "repeat": 3, "concurrency": 2, "requested_duration": 20, "elapsed_seconds": 20.969, "warmup_completed": 1, "completed": 40, "successful_rps": 1.908, "failures": [], "error_fraction": 0.0, "p50": 1.0675, "p95": 1.2299, "p99": 1.3224, "upload_p95": 0.2471} diff --git a/evidence/load.png b/evidence/load.png new file mode 100644 index 00000000..8e504b94 Binary files /dev/null and b/evidence/load.png differ diff --git a/evidence/measurements.csv b/evidence/measurements.csv new file mode 100644 index 00000000..ae5f9105 --- /dev/null +++ b/evidence/measurements.csv @@ -0,0 +1,7 @@ +profile,repeat,completed,successful_rps,p50,p95,p99,error_fraction +320x200,1,80,3.982,0.401,0.7265,0.7436,0.0 +320x200,2,73,3.607,0.4219,0.7366,0.7857,0.0 +320x200,3,71,3.549,0.4389,0.7694,0.82,0.0 +1024x768,1,45,2.188,0.8308,1.1487,1.1615,0.0 +1024x768,2,32,1.569,0.79,1.1397,1.2381,0.030303030303030304 +1024x768,3,40,1.908,1.0675,1.2299,1.3224,0.0 diff --git a/evidence/normal-start-retry.txt b/evidence/normal-start-retry.txt new file mode 100644 index 00000000..b6177cfb --- /dev/null +++ b/evidence/normal-start-retry.txt @@ -0,0 +1,37 @@ + Container lyakhonov-lab5-redis-1 Running + Container lyakhonov-lab5-s3-1 Running + Container lyakhonov-lab5-db-1 Running + Container lyakhonov-lab5-broker-1 Running + Container lyakhonov-lab5-dispatcher-1 Running + Container lyakhonov-lab5-staging-cleaner-1 Running + Container lyakhonov-lab5-worker-1 Running + Container lyakhonov-lab5-worker-2 Running + Container lyakhonov-lab5-api-1 Running + Container lyakhonov-lab5-api-2 Running + Container lyakhonov-lab5-lb-1 Running + Container lyakhonov-lab5-bucket-init-1 Starting + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Started + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Starting + Container lyakhonov-lab5-migrate-1 Started + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-broker-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-broker-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-broker-1 Healthy + Container lyakhonov-lab5-broker-1 Healthy + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited diff --git a/evidence/normal-start.txt b/evidence/normal-start.txt new file mode 100644 index 00000000..4e3c8816 --- /dev/null +++ b/evidence/normal-start.txt @@ -0,0 +1,80 @@ + Container lyakhonov-lab5-redis-1 Running + Container lyakhonov-lab5-broker-1 Running + Container lyakhonov-lab5-db-1 Running + Container lyakhonov-lab5-s3-1 Running + Container lyakhonov-lab5-migrate-1 Recreate + Container lyakhonov-lab5-migrate-1 Recreated + Container lyakhonov-lab5-staging-cleaner-1 Recreate + Container lyakhonov-lab5-dispatcher-1 Recreate + Container lyakhonov-lab5-worker-2 Recreate + Container lyakhonov-lab5-api-2 Recreate + Container lyakhonov-lab5-api-1 Recreate + Container lyakhonov-lab5-worker-1 Recreate + Container lyakhonov-lab5-worker-2 Recreated + Container lyakhonov-lab5-worker-1 Recreated + Container lyakhonov-lab5-dispatcher-1 Recreated + Container lyakhonov-lab5-api-2 Recreated + Container lyakhonov-lab5-api-1 Recreated + Container lyakhonov-lab5-lb-1 Running + Container lyakhonov-lab5-staging-cleaner-1 Recreated + Container lyakhonov-lab5-bucket-init-1 Starting + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Started + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Starting + Container lyakhonov-lab5-migrate-1 Started + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-broker-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-broker-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-broker-1 Healthy + Container lyakhonov-lab5-broker-1 Healthy + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-staging-cleaner-1 Starting + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-worker-2 Starting + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-dispatcher-1 Starting + Container lyakhonov-lab5-api-1 Starting + Container lyakhonov-lab5-api-1 Started + Container lyakhonov-lab5-api-2 Starting + Container lyakhonov-lab5-staging-cleaner-1 Started + Container lyakhonov-lab5-dispatcher-1 Started + Container lyakhonov-lab5-worker-2 Started + Container lyakhonov-lab5-worker-1 Starting + Container lyakhonov-lab5-api-2 Started + Container lyakhonov-lab5-worker-1 Started + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-worker-2 Waiting + Container lyakhonov-lab5-worker-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-s3-1 Waiting + Container lyakhonov-lab5-broker-1 Waiting + Container lyakhonov-lab5-lb-1 Waiting + Container lyakhonov-lab5-dispatcher-1 Waiting + Container lyakhonov-lab5-staging-cleaner-1 Waiting + Container lyakhonov-lab5-api-1 Waiting + Container lyakhonov-lab5-api-2 Waiting + Container lyakhonov-lab5-redis-1 Waiting + Container lyakhonov-lab5-lb-1 Healthy + Container lyakhonov-lab5-redis-1 Healthy + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-s3-1 Healthy + Container lyakhonov-lab5-broker-1 Healthy + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-api-1 Healthy + Container lyakhonov-lab5-api-2 Healthy +container lyakhonov-lab5-worker-2 has no healthcheck configured diff --git a/evidence/persistence.json b/evidence/persistence.json new file mode 100644 index 00000000..0753eeca --- /dev/null +++ b/evidence/persistence.json @@ -0,0 +1,176 @@ +{ + "command": [ + "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-X", + "utf8", + "tests/resilience.py", + "--mode", + "persistence", + "--base-url", + "http://127.0.0.1:8085", + "--action-script", + "C:\\Users\\User\\Desktop\\vysoko-nagr 5\\scripts\\restart.cmd" + ], + "exit_code": 0, + "output": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.\n", + "before": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:56:56.938529239Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:56:57.198039404Z", + "running": true + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:56:31.315056049Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:58:21.664983091Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:56:56.914519489Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:58:22.174853427Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:56:56.92548889Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:56:57.356659637Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:56:56.927116397Z", + "running": true + } + ], + "after": [ + { + "id": "b3f8a331a59832dde80002af9f29c5893381e780803283476c687168ede15842", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:58:31.589874697Z", + "running": true + }, + { + "id": "a0acb99b68a1f803792742460827528137ac3ab96ec76a68de550f930a4a9368", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:58:31.785178375Z", + "running": true + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:58:31.88055044Z", + "running": true + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:58:21.664983091Z", + "running": false + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:58:30.504381937Z", + "running": true + }, + { + "id": "bdedd08309bb0db9dc94235a148fc0938180449b1431059178488d0c6f985049", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:58:31.312524166Z", + "running": true + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:58:30.862509137Z", + "running": true + }, + { + "id": "39b8b11f3c7c9e1c3fb778e780dc8d123e4b955669ddee8c85420eefcb3abaca", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:58:22.174853427Z", + "running": false + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:58:30.614258194Z", + "running": true + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:58:31.091098179Z", + "running": true + }, + { + "id": "23b814fb71a813bedb09712be57c736e19460c2c25d2b2b4b0e66facb0bfdab9", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:58:31.667560755Z", + "running": true + }, + { + "id": "1f724278797561aea476251a8902eac6377039c3e30cbc336d501409af118ae2", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:58:31.445500705Z", + "running": true + }, + { + "id": "e36e30c0bc23ffdc3c2c4fe39b57ea98597764dfa313c509bd1734d74b801b44", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:58:31.518418339Z", + "running": true + } + ] +} \ No newline at end of file diff --git a/evidence/pipeline-initial-run.txt b/evidence/pipeline-initial-run.txt new file mode 100644 index 00000000..9e7cbf46 --- /dev/null +++ b/evidence/pipeline-initial-run.txt @@ -0,0 +1,56 @@ +PASS: negative_upload_acl_and_24_concurrent_messages +PASS: accepted_survives_api_broker_restart +PASS: before_publish +PASS: after_publish +PASS: before_effect +PASS: after_original +PASS: after_variants +PASS: after_commit +PASS: before_publish +PASS: after_publish +PASS: before_effect +PASS: after_original +PASS: after_variants +PASS: after_commit +PASS: before_publish +PASS: after_publish +PASS: before_effect +PASS: after_original +PASS: after_variants +PASS: after_commit +Traceback (most recent call last): + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 234, in + Experiment(parser.parse_args()).main() + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 222, in main + self.versions() + ~~~~~~~~~~~~~^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 135, in versions + objects = self.probe('objects', id=att['id']) + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 35, in probe + check(result.returncode == 0, 'probe failed: ' + result.stderr) + ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\tests\client.py", line 24, in check + raise Failure(message) +client.Failure: probe failed: Traceback (most recent call last): + File "", line 54, in + File "", line 35, in run + File "/usr/local/lib/python3.11/site-packages/botocore/paginate.py", line 272, in __iter__ + response = self._make_request(current_kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/context.py", line 123, in wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/paginate.py", line 360, in _make_request + return self._method(**current_kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/client.py", line 602, in _api_call + return self._make_api_call(operation_name, kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/context.py", line 123, in wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/client.py", line 1078, in _make_api_call + raise error_class(parsed_response, operation_name) +botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the ListObjectsV2 operation: Access Denied. + diff --git a/evidence/pipeline-resumed-run.txt b/evidence/pipeline-resumed-run.txt new file mode 100644 index 00000000..a8fec748 --- /dev/null +++ b/evidence/pipeline-resumed-run.txt @@ -0,0 +1,8 @@ +PASS: stale_running_worker_and_reordered_duplicates +PASS: upgrade_ready_after_staging_cleanup +PASS: stale_running_worker_and_reordered_duplicates +PASS: upgrade_ready_after_staging_cleanup +PASS: stale_running_worker_and_reordered_duplicates +PASS: upgrade_ready_after_staging_cleanup +PASS: permanent_error_and_explicit_replay +PASS: transient_error_and_explicit_replay diff --git a/evidence/pipeline-run.txt b/evidence/pipeline-run.txt new file mode 100644 index 00000000..9e7cbf46 --- /dev/null +++ b/evidence/pipeline-run.txt @@ -0,0 +1,56 @@ +PASS: negative_upload_acl_and_24_concurrent_messages +PASS: accepted_survives_api_broker_restart +PASS: before_publish +PASS: after_publish +PASS: before_effect +PASS: after_original +PASS: after_variants +PASS: after_commit +PASS: before_publish +PASS: after_publish +PASS: before_effect +PASS: after_original +PASS: after_variants +PASS: after_commit +PASS: before_publish +PASS: after_publish +PASS: before_effect +PASS: after_original +PASS: after_variants +PASS: after_commit +Traceback (most recent call last): + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 234, in + Experiment(parser.parse_args()).main() + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 222, in main + self.versions() + ~~~~~~~~~~~~~^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 135, in versions + objects = self.probe('objects', id=att['id']) + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\pipeline.py", line 35, in probe + check(result.returncode == 0, 'probe failed: ' + result.stderr) + ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\tests\client.py", line 24, in check + raise Failure(message) +client.Failure: probe failed: Traceback (most recent call last): + File "", line 54, in + File "", line 35, in run + File "/usr/local/lib/python3.11/site-packages/botocore/paginate.py", line 272, in __iter__ + response = self._make_request(current_kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/context.py", line 123, in wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/paginate.py", line 360, in _make_request + return self._method(**current_kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/client.py", line 602, in _api_call + return self._make_api_call(operation_name, kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/context.py", line 123, in wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/botocore/client.py", line 1078, in _make_api_call + raise error_class(parsed_response, operation_name) +botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the ListObjectsV2 operation: Access Denied. + diff --git a/evidence/pipeline.json b/evidence/pipeline.json new file mode 100644 index 00000000..37fcdcdb --- /dev/null +++ b/evidence/pipeline.json @@ -0,0 +1,5145 @@ +[ + { + "scenario": "negative_upload_acl_and_24_concurrent_messages", + "seconds": 4.168, + "checks": 9, + "message_ids": 1 + }, + { + "scenario": "accepted_survives_api_broker_restart", + "seconds": 27.593, + "attachment_id": "619fa63f-4fc4-4022-9ebb-d45ac01ac375", + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:41:58.297145512Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:41:58.683174598Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:39:52.157659158Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:41:55.56150084Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:41:58.280887966Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:41:56.049697978Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:41:58.292710818Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:41:58.616417234Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:42:17.221143902Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:41:58.280887966Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:42:17.723111805Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:42:33.06126539Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:42:33.37189925Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "before_publish", + "seconds": 32.205, + "repeat": 1, + "attachment_id": "de278c50-0dea-4db7-bbc5-633e03ae90ad", + "fault": { + "job_id": "5112c4f0-fee6-4814-bf47-09f0408f5b7d", + "point": "before_publish", + "action": "crash", + "hit_at": "2026-11-20 03:42:50.467465+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "5112c4f0-fee6-4814-bf47-09f0408f5b7d", + "attachment_id": "de278c50-0dea-4db7-bbc5-633e03ae90ad", + "staging_key": "staging/de278c50-0dea-4db7-bbc5-633e03ae90ad", + "created_at": "2026-11-20 03:42:39.379658+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:43:05.529083+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:43:05.762115+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:42:17.221143902Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:41:58.280887966Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:42:17.723111805Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:42:33.06126539Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:42:33.37189925Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:42:44.051926228Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:42:50.690235738Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:42:44.545702483Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:42:33.06126539Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:42:33.37189925Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_publish", + "seconds": 16.65, + "repeat": 1, + "attachment_id": "132c66de-7282-45a3-a200-11d33b54e2ed", + "fault": { + "job_id": "df29a891-03c2-49e2-990c-1cd835743ea0", + "point": "after_publish", + "action": "crash", + "hit_at": "2026-11-20 03:43:20.963939+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "df29a891-03c2-49e2-990c-1cd835743ea0", + "attachment_id": "132c66de-7282-45a3-a200-11d33b54e2ed", + "staging_key": "staging/132c66de-7282-45a3-a200-11d33b54e2ed", + "created_at": "2026-11-20 03:43:11.552216+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": null, + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:43:21.503078+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:42:44.051926228Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:42:50.690235738Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:42:44.545702483Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:42:33.06126539Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:42:33.37189925Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:43:16.580731829Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:43:17.066108428Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:42:33.06126539Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:42:33.37189925Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "before_effect", + "seconds": 28.623, + "repeat": 1, + "attachment_id": "ec2c7c0c-6a39-4715-944e-8faf09b2279f", + "fault": { + "job_id": "06dbadb4-c510-4c08-b28d-5a0dd00a5b02", + "point": "before_effect", + "action": "crash", + "hit_at": "2026-11-20 03:43:37.411076+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "06dbadb4-c510-4c08-b28d-5a0dd00a5b02", + "attachment_id": "ec2c7c0c-6a39-4715-944e-8faf09b2279f", + "staging_key": "staging/ec2c7c0c-6a39-4715-944e-8faf09b2279f", + "created_at": "2026-11-20 03:43:28.547524+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:43:49.044160+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:43:49.465850+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:43:16.580731829Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:43:17.066108428Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:42:33.06126539Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:42:33.37189925Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:43:33.084987014Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:43:33.552597501Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:43:37.634736588Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:43:36.683144593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_original", + "seconds": 26.509, + "repeat": 1, + "attachment_id": "62f6f8a5-a3c2-4a5b-ad26-d035b9511998", + "fault": { + "job_id": "55a16d30-7c35-454c-86fd-d689757a51e6", + "point": "after_original", + "action": "crash", + "hit_at": "2026-11-20 03:44:05.931398+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "55a16d30-7c35-454c-86fd-d689757a51e6", + "attachment_id": "62f6f8a5-a3c2-4a5b-ad26-d035b9511998", + "staging_key": "staging/62f6f8a5-a3c2-4a5b-ad26-d035b9511998", + "created_at": "2026-11-20 03:43:57.216845+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:44:17.192528+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:44:17.509618+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:43:33.084987014Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:43:33.552597501Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:43:37.634736588Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:43:36.683144593Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:44:01.486596866Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:44:01.983245268Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:44:06.210667354Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:04.953115416Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_variants", + "seconds": 27.131, + "repeat": 1, + "attachment_id": "78bb9cfa-55c4-46d0-bc39-548f8311fbd0", + "fault": { + "job_id": "3a5263bf-c4e3-4516-abc7-7e5e2b8da5e4", + "point": "after_variants", + "action": "crash", + "hit_at": "2026-11-20 03:44:32.808598+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "3a5263bf-c4e3-4516-abc7-7e5e2b8da5e4", + "attachment_id": "78bb9cfa-55c4-46d0-bc39-548f8311fbd0", + "staging_key": "staging/78bb9cfa-55c4-46d0-bc39-548f8311fbd0", + "created_at": "2026-11-20 03:44:23.725433+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:44:43.705309+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:44:43.969458+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:44:01.486596866Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:44:01.983245268Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:44:06.210667354Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:04.953115416Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:44:27.912594847Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:44:28.369270089Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:44:33.022103249Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:31.933900075Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_commit", + "seconds": 17.081, + "repeat": 1, + "attachment_id": "352b23af-8c50-4927-bf13-70a1bb6fce21", + "fault": { + "job_id": "ecf012e7-06b4-4ae4-a45d-70028bfc12e4", + "point": "after_commit", + "action": "crash", + "hit_at": "2026-11-20 03:44:59.812097+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "ecf012e7-06b4-4ae4-a45d-70028bfc12e4", + "attachment_id": "352b23af-8c50-4927-bf13-70a1bb6fce21", + "staging_key": "staging/352b23af-8c50-4927-bf13-70a1bb6fce21", + "created_at": "2026-11-20 03:44:50.799790+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:44:50.989770+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:44:59.810079+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:44:27.912594847Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:44:28.369270089Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:44:33.022103249Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:31.933900075Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:44:55.193771438Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:44:55.695401835Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:00.069022774Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:58.720463171Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "before_publish", + "seconds": 28.94, + "repeat": 2, + "attachment_id": "dd45ec28-ca62-4702-83cc-0ad183fa73f0", + "fault": { + "job_id": "055f56e9-86fd-42c9-8b20-5dc72011918b", + "point": "before_publish", + "action": "crash", + "hit_at": "2026-11-20 03:45:15.188522+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "055f56e9-86fd-42c9-8b20-5dc72011918b", + "attachment_id": "dd45ec28-ca62-4702-83cc-0ad183fa73f0", + "staging_key": "staging/dd45ec28-ca62-4702-83cc-0ad183fa73f0", + "created_at": "2026-11-20 03:45:07.608313+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:45:30.320391+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:45:30.632786+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:44:55.193771438Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:43:21.198449094Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:44:55.695401835Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:00.069022774Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:58.720463171Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:45:11.239538513Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:15.431620016Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:45:11.825599429Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:00.069022774Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:58.720463171Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_publish", + "seconds": 15.182, + "repeat": 2, + "attachment_id": "eaf40096-5053-4ef3-a9db-14dfc234a8ce", + "fault": { + "job_id": "d4c4afd1-7e37-447d-8c41-2ebad3bca874", + "point": "after_publish", + "action": "crash", + "hit_at": "2026-11-20 03:45:43.998330+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "d4c4afd1-7e37-447d-8c41-2ebad3bca874", + "attachment_id": "eaf40096-5053-4ef3-a9db-14dfc234a8ce", + "staging_key": "staging/eaf40096-5053-4ef3-a9db-14dfc234a8ce", + "created_at": "2026-11-20 03:45:36.404146+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": null, + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:45:44.388768+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:45:11.239538513Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:15.431620016Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:45:11.825599429Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:00.069022774Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:58.720463171Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:45:40.152387954Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:45:40.655870854Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:00.069022774Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:58.720463171Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "before_effect", + "seconds": 25.678, + "repeat": 2, + "attachment_id": "1949b244-780e-4a3e-8be2-918310eb0854", + "fault": { + "job_id": "eb32b287-eed7-4336-8435-a170219d0751", + "point": "before_effect", + "action": "crash", + "hit_at": "2026-11-20 03:45:59.502654+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "eb32b287-eed7-4336-8435-a170219d0751", + "attachment_id": "1949b244-780e-4a3e-8be2-918310eb0854", + "staging_key": "staging/1949b244-780e-4a3e-8be2-918310eb0854", + "created_at": "2026-11-20 03:45:51.858645+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:46:11.045887+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:46:11.339887+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:45:40.152387954Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:45:40.655870854Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:00.069022774Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:44:58.720463171Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:45:56.13119465Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:45:56.615857607Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:59.755318954Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:45:58.61301125Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_original", + "seconds": 25.642, + "repeat": 2, + "attachment_id": "85b5126d-f0e3-499d-aca8-6c13e1818e28", + "fault": { + "job_id": "bd85defc-0084-4d19-b20f-2421363745b1", + "point": "after_original", + "action": "crash", + "hit_at": "2026-11-20 03:46:25.817634+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "bd85defc-0084-4d19-b20f-2421363745b1", + "attachment_id": "85b5126d-f0e3-499d-aca8-6c13e1818e28", + "staging_key": "staging/85b5126d-f0e3-499d-aca8-6c13e1818e28", + "created_at": "2026-11-20 03:46:17.558732+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:46:36.887135+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:46:37.193120+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:45:56.13119465Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:45:56.615857607Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:45:59.755318954Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:45:58.61301125Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:46:21.542059999Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:46:22.037531019Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:46:26.027818995Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:46:24.617098765Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_variants", + "seconds": 25.458, + "repeat": 2, + "attachment_id": "b84b0b1f-3265-494d-a179-9f71daa68613", + "fault": { + "job_id": "ac944804-f74f-4ce5-9d25-22c4565b7887", + "point": "after_variants", + "action": "crash", + "hit_at": "2026-11-20 03:46:51.060205+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "ac944804-f74f-4ce5-9d25-22c4565b7887", + "attachment_id": "b84b0b1f-3265-494d-a179-9f71daa68613", + "staging_key": "staging/b84b0b1f-3265-494d-a179-9f71daa68613", + "created_at": "2026-11-20 03:46:43.356889+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:47:02.105387+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:47:02.403063+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:46:21.542059999Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:46:22.037531019Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:46:26.027818995Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:46:24.617098765Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:46:47.067326151Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:46:47.551997225Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:46:51.298628199Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:46:49.994100608Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_commit", + "seconds": 14.989, + "repeat": 2, + "attachment_id": "8e07fb74-6f9d-4cb1-a79e-1b791fc70534", + "fault": { + "job_id": "1f4c65ab-6330-4b9e-b2f0-d661ba79b963", + "point": "after_commit", + "action": "crash", + "hit_at": "2026-11-20 03:47:16.519594+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "1f4c65ab-6330-4b9e-b2f0-d661ba79b963", + "attachment_id": "8e07fb74-6f9d-4cb1-a79e-1b791fc70534", + "staging_key": "staging/8e07fb74-6f9d-4cb1-a79e-1b791fc70534", + "created_at": "2026-11-20 03:47:08.839253+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:47:09.075864+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:47:16.517978+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:46:47.067326151Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:46:47.551997225Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:46:51.298628199Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:46:49.994100608Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:47:12.483349362Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:47:12.913604465Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:47:16.742277702Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:47:15.490450932Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "before_publish", + "seconds": 29.804, + "repeat": 3, + "attachment_id": "d75d929a-6d7f-466b-b26f-7e72c2407a5a", + "fault": { + "job_id": "dd9ba48e-f4b6-439e-b6f1-82e3400d3538", + "point": "before_publish", + "action": "crash", + "hit_at": "2026-11-20 03:47:31.744254+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "dd9ba48e-f4b6-439e-b6f1-82e3400d3538", + "attachment_id": "d75d929a-6d7f-466b-b26f-7e72c2407a5a", + "staging_key": "staging/d75d929a-6d7f-466b-b26f-7e72c2407a5a", + "created_at": "2026-11-20 03:47:23.446790+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:47:46.792588+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:47:47.081130+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:47:12.483349362Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:45:44.20441819Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:47:12.913604465Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:47:16.742277702Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:47:15.490450932Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:47:27.559641991Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:47:31.929204122Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:47:28.057242867Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:47:16.742277702Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:47:15.490450932Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_publish", + "seconds": 14.478, + "repeat": 3, + "attachment_id": "0306a14e-ff85-4e84-8455-2802974bd23e", + "fault": { + "job_id": "5a36ea65-1082-4564-8bd9-aeb4bebc3cf3", + "point": "after_publish", + "action": "crash", + "hit_at": "2026-11-20 03:48:01.004583+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "5a36ea65-1082-4564-8bd9-aeb4bebc3cf3", + "attachment_id": "0306a14e-ff85-4e84-8455-2802974bd23e", + "staging_key": "staging/0306a14e-ff85-4e84-8455-2802974bd23e", + "created_at": "2026-11-20 03:47:53.380497+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": null, + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:48:01.416005+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:47:27.559641991Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:47:31.929204122Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:47:28.057242867Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:47:16.742277702Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:47:15.490450932Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:47:57.217828378Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:47:57.690157191Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:47:16.742277702Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:47:15.490450932Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "before_effect", + "seconds": 26.598, + "repeat": 3, + "attachment_id": "606f90f6-9e67-4082-b961-5b2bca12ec2a", + "fault": { + "job_id": "d90be4cb-5786-4553-b770-c25e59fdcd18", + "point": "before_effect", + "action": "crash", + "hit_at": "2026-11-20 03:48:16.197409+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "d90be4cb-5786-4553-b770-c25e59fdcd18", + "attachment_id": "606f90f6-9e67-4082-b961-5b2bca12ec2a", + "staging_key": "staging/606f90f6-9e67-4082-b961-5b2bca12ec2a", + "created_at": "2026-11-20 03:48:08.004079+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:48:28.023266+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:48:28.383964+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:47:57.217828378Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:47:57.690157191Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:47:16.742277702Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:47:15.490450932Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:48:12.202153026Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:48:12.697238174Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:48:16.424435299Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:48:15.687396384Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_original", + "seconds": 26.292, + "repeat": 3, + "attachment_id": "917d592f-eb5f-4c07-8a2b-d5020414c012", + "fault": { + "job_id": "0d555a0c-4c0e-48bc-8b5e-001b0e78a2c8", + "point": "after_original", + "action": "crash", + "hit_at": "2026-11-20 03:48:42.768054+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "0d555a0c-4c0e-48bc-8b5e-001b0e78a2c8", + "attachment_id": "917d592f-eb5f-4c07-8a2b-d5020414c012", + "staging_key": "staging/917d592f-eb5f-4c07-8a2b-d5020414c012", + "created_at": "2026-11-20 03:48:34.502820+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:48:54.167617+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:48:54.418755+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:48:12.202153026Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:48:12.697238174Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:48:16.424435299Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:48:15.687396384Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:48:38.330040227Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:48:38.829333153Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:48:43.018521035Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:48:41.772811802Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_variants", + "seconds": 25.466, + "repeat": 3, + "attachment_id": "5229c691-7f61-445e-8922-47c8dcb92b99", + "fault": { + "job_id": "f46470fb-d143-4c09-a40c-b05674b33e69", + "point": "after_variants", + "action": "crash", + "hit_at": "2026-11-20 03:49:08.669526+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "f46470fb-d143-4c09-a40c-b05674b33e69", + "attachment_id": "5229c691-7f61-445e-8922-47c8dcb92b99", + "staging_key": "staging/5229c691-7f61-445e-8922-47c8dcb92b99", + "created_at": "2026-11-20 03:49:00.638699+00:00", + "attempt": 2, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:49:19.711271+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:49:20.069410+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:48:38.330040227Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:48:38.829333153Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:48:43.018521035Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:48:41.772811802Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:49:04.391235089Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:49:04.865540417Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:49:08.946091975Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:49:07.317159304Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "after_commit", + "seconds": 16.739, + "repeat": 3, + "attachment_id": "32278609-0542-4c9e-9ddf-7238d78966f7", + "fault": { + "job_id": "5b89ba7a-caa6-4364-ad27-befa2904c313", + "point": "after_commit", + "action": "crash", + "hit_at": "2026-11-20 03:49:35.327700+00:00", + "process_id": 1, + "released": false + }, + "job_after": { + "id": "5b89ba7a-caa6-4364-ad27-befa2904c313", + "attachment_id": "32278609-0542-4c9e-9ddf-7238d78966f7", + "staging_key": "staging/32278609-0542-4c9e-9ddf-7238d78966f7", + "created_at": "2026-11-20 03:49:26.526430+00:00", + "attempt": 1, + "attempt_limit": 5, + "state": "ready", + "next_publish_at": null, + "published_at": "2026-11-20 03:49:26.589812+00:00", + "publish_failures": 0, + "lease_until": null, + "lease_token": null, + "last_error": null, + "pipeline_version": 1, + "settled_at": "2026-11-20 03:49:35.325775+00:00" + }, + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:49:04.391235089Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:49:04.865540417Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:49:08.946091975Z", + "running": false, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:49:07.317159304Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:49:30.614157138Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:49:31.125905282Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:49:35.57674716Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:49:34.180088959Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + }, + { + "scenario": "stale_running_worker_and_reordered_duplicates", + "seconds": 45.964, + "repeat": 1, + "attachment_id": "bd690fb9-1393-4fd8-94fe-0bcc4c78f632", + "before_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + }, + "after_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + }, + "versions": [ + { + "pipeline_version": 1, + "state": "superseded", + "attempt": 1 + }, + { + "pipeline_version": 2, + "state": "ready", + "attempt": 1 + } + ], + "objects": [ + { + "key": "originals/bd690fb9-1393-4fd8-94fe-0bcc4c78f632", + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "size": 111817 + }, + { + "key": "variants/bd690fb9-1393-4fd8-94fe-0bcc4c78f632/v1/crop", + "etag": "\"989d73cfe100e73f9b13d8260e1c2e30\"", + "size": 2462 + }, + { + "key": "variants/bd690fb9-1393-4fd8-94fe-0bcc4c78f632/v1/optimized", + "etag": "\"14bfc73d33bbab974088b27794a8e34a\"", + "size": 2576 + }, + { + "key": "variants/bd690fb9-1393-4fd8-94fe-0bcc4c78f632/v1/thumbnail", + "etag": "\"e31836875f24868250c5fc4469d1c8d0\"", + "size": 5813 + }, + { + "key": "variants/bd690fb9-1393-4fd8-94fe-0bcc4c78f632/v2/crop", + "etag": "\"f377b4ca5f884179377dc3189b8e10af\"", + "size": 2888 + }, + { + "key": "variants/bd690fb9-1393-4fd8-94fe-0bcc4c78f632/v2/optimized", + "etag": "\"4e41ece95aee8e9fdf8097022e10da9e\"", + "size": 3152 + }, + { + "key": "variants/bd690fb9-1393-4fd8-94fe-0bcc4c78f632/v2/thumbnail", + "etag": "\"85e637fcc90201e8a3a98996b4025373\"", + "size": 7014 + } + ] + }, + { + "scenario": "upgrade_ready_after_staging_cleanup", + "seconds": 11.151, + "repeat": 1, + "attachment_id": "77489ca8-15ee-4b8d-a21f-d310454650e6", + "before_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "crop": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "optimized": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c" + }, + "after_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + } + }, + { + "scenario": "stale_running_worker_and_reordered_duplicates", + "seconds": 45.456, + "repeat": 2, + "attachment_id": "3ce6bb5f-9194-451c-b27e-02a6ec63881a", + "before_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + }, + "after_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + }, + "versions": [ + { + "pipeline_version": 1, + "state": "superseded", + "attempt": 1 + }, + { + "pipeline_version": 2, + "state": "ready", + "attempt": 1 + } + ], + "objects": [ + { + "key": "originals/3ce6bb5f-9194-451c-b27e-02a6ec63881a", + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "size": 111817 + }, + { + "key": "variants/3ce6bb5f-9194-451c-b27e-02a6ec63881a/v1/crop", + "etag": "\"989d73cfe100e73f9b13d8260e1c2e30\"", + "size": 2462 + }, + { + "key": "variants/3ce6bb5f-9194-451c-b27e-02a6ec63881a/v1/optimized", + "etag": "\"14bfc73d33bbab974088b27794a8e34a\"", + "size": 2576 + }, + { + "key": "variants/3ce6bb5f-9194-451c-b27e-02a6ec63881a/v1/thumbnail", + "etag": "\"e31836875f24868250c5fc4469d1c8d0\"", + "size": 5813 + }, + { + "key": "variants/3ce6bb5f-9194-451c-b27e-02a6ec63881a/v2/crop", + "etag": "\"f377b4ca5f884179377dc3189b8e10af\"", + "size": 2888 + }, + { + "key": "variants/3ce6bb5f-9194-451c-b27e-02a6ec63881a/v2/optimized", + "etag": "\"4e41ece95aee8e9fdf8097022e10da9e\"", + "size": 3152 + }, + { + "key": "variants/3ce6bb5f-9194-451c-b27e-02a6ec63881a/v2/thumbnail", + "etag": "\"85e637fcc90201e8a3a98996b4025373\"", + "size": 7014 + } + ] + }, + { + "scenario": "upgrade_ready_after_staging_cleanup", + "seconds": 11.485, + "repeat": 2, + "attachment_id": "563c45e8-537d-4634-a1c3-0037468836f9", + "before_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "crop": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "optimized": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c" + }, + "after_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + } + }, + { + "scenario": "stale_running_worker_and_reordered_duplicates", + "seconds": 64.377, + "repeat": 3, + "attachment_id": "7117c2f5-48d1-4227-afa0-a2ee439a067e", + "before_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + }, + "after_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + }, + "versions": [ + { + "pipeline_version": 1, + "state": "superseded", + "attempt": 1 + }, + { + "pipeline_version": 2, + "state": "ready", + "attempt": 1 + } + ], + "objects": [ + { + "key": "originals/7117c2f5-48d1-4227-afa0-a2ee439a067e", + "etag": "\"c2ab21400d8c13ea847e4d5d3c9832ad\"", + "size": 111817 + }, + { + "key": "variants/7117c2f5-48d1-4227-afa0-a2ee439a067e/v1/crop", + "etag": "\"989d73cfe100e73f9b13d8260e1c2e30\"", + "size": 2462 + }, + { + "key": "variants/7117c2f5-48d1-4227-afa0-a2ee439a067e/v1/optimized", + "etag": "\"14bfc73d33bbab974088b27794a8e34a\"", + "size": 2576 + }, + { + "key": "variants/7117c2f5-48d1-4227-afa0-a2ee439a067e/v1/thumbnail", + "etag": "\"e31836875f24868250c5fc4469d1c8d0\"", + "size": 5813 + }, + { + "key": "variants/7117c2f5-48d1-4227-afa0-a2ee439a067e/v2/crop", + "etag": "\"f377b4ca5f884179377dc3189b8e10af\"", + "size": 2888 + }, + { + "key": "variants/7117c2f5-48d1-4227-afa0-a2ee439a067e/v2/optimized", + "etag": "\"4e41ece95aee8e9fdf8097022e10da9e\"", + "size": 3152 + }, + { + "key": "variants/7117c2f5-48d1-4227-afa0-a2ee439a067e/v2/thumbnail", + "etag": "\"85e637fcc90201e8a3a98996b4025373\"", + "size": 7014 + } + ] + }, + { + "scenario": "upgrade_ready_after_staging_cleanup", + "seconds": 15.628, + "repeat": 3, + "attachment_id": "04003138-9d81-4a15-b6d9-c16a12a79228", + "before_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "crop": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "optimized": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c" + }, + "after_digest": { + "original": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "thumbnail": "79e8eafd7e4c9a54cbdd67620a7ba3507daca6aff179c5105cbb082ff5b95ef0", + "crop": "7eeb30cabd9c5725a906d94701898aee61205fc0e1dd7cd3dc83c9d559f2b566", + "optimized": "9f45b82df9622236a018966f0274f08eab75b162e31bffa68db3647b95105759" + } + }, + { + "scenario": "permanent_error_and_explicit_replay", + "seconds": 22.401, + "attachment_id": "77a4b759-5a49-4633-8152-1038fe144ce7", + "attempts": [ + { + "attempt": 1, + "state": "failed", + "next_publish_at": null, + "last_error": "staging_integrity" + } + ], + "audit": [ + { + "attempt": 2, + "reason": "operator" + } + ] + }, + { + "scenario": "transient_error_and_explicit_replay", + "seconds": 44.885, + "attachment_id": "6601079a-655c-4827-9174-b2b1c57685d2", + "attempts": [ + { + "attempt": 2, + "state": "queued", + "next_publish_at": "2026-11-20 03:55:16.096457+00:00", + "last_error": "processing_failed" + }, + { + "attempt": 4, + "state": "queued", + "next_publish_at": "2026-11-20 03:55:22.676112+00:00", + "last_error": "processing_failed" + }, + { + "attempt": 5, + "state": "queued", + "next_publish_at": "2026-11-20 03:55:30.492215+00:00", + "last_error": "processing_failed" + }, + { + "attempt": 5, + "state": "queued", + "next_publish_at": "2026-11-20 03:55:30.492215+00:00", + "last_error": "processing_failed" + }, + { + "attempt": 5, + "state": "queued", + "next_publish_at": "2026-11-20 03:55:30.492215+00:00", + "last_error": "processing_failed" + }, + { + "attempt": 5, + "state": "failed", + "next_publish_at": null, + "last_error": "processing_failed" + } + ], + "audit": [ + { + "attempt": 6, + "reason": "operator" + } + ] + }, + { + "scenario": "negative_upload_acl_and_24_concurrent_messages", + "seconds": 12.022, + "checks": 9, + "message_ids": 1 + }, + { + "scenario": "accepted_survives_api_broker_restart", + "seconds": 35.415, + "attachment_id": "1450e655-aebb-4b55-8196-7823df0fb275", + "before": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:42:15.518481555Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:42:15.729148361Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:42:16.413919787Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:55:53.440888182Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:55:54.012916611Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:55:13.450036277Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:55:13.735876724Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + } + ], + "after": [ + { + "id": "86d8b62b07da61b7b8539094d65f6712ffc0a342cd811b9ec12b6c302fb9e992", + "name": "/lyakhonov-lab5-api-1", + "started_at": "2026-11-20T03:56:30.629418613Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "75c9985d99c323b3db7e20f01ecc7d50e8ee32e1a10ddccc17984eb0c49d0902", + "name": "/lyakhonov-lab5-api-2", + "started_at": "2026-11-20T03:56:30.739564284Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "b94a25ac1bb3df998e3d0c27beae4b97d655bbc0a7ae520bd60c736cd40462c9", + "name": "/lyakhonov-lab5-broker-1", + "started_at": "2026-11-20T03:56:31.315056049Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "fdca00783fcb86e75d3ec4c84494d1571548333d585459191b78992f707729ad", + "name": "/lyakhonov-lab5-bucket-init-1", + "started_at": "2026-11-20T03:56:32.571405318Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "10408d41f3b50b0a5e8ad4e21b849fa97548c03f4a535b27c8c919163b9c30b2", + "name": "/lyakhonov-lab5-db-1", + "started_at": "2026-11-20T03:39:52.15355593Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "357e0877d3d2ad9692375f8063842b02b920271612671a418e383ff6af93e21a", + "name": "/lyakhonov-lab5-dispatcher-1", + "started_at": "2026-11-20T03:48:01.229241426Z", + "running": true, + "restart_count": 1, + "exit_code": 0 + }, + { + "id": "934fcb61b954722ec6f9d3ce84d3399f5a35a30e8368e0ed44a939319ea9ff4f", + "name": "/lyakhonov-lab5-lb-1", + "started_at": "2026-11-20T03:39:59.703127904Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "4672ce428bbbe0cad051b0b4713019cc155fc801ecdf1480d5a8234ddfa160ef", + "name": "/lyakhonov-lab5-migrate-1", + "started_at": "2026-11-20T03:56:33.084907991Z", + "running": false, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "c3962c31749b802835f909ce1b4efbe34149e88d9bc678ca3b82f9fdbcc97af1", + "name": "/lyakhonov-lab5-redis-1", + "started_at": "2026-11-20T03:39:52.151204203Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "0b09dce985685e130ed8774a277a3ff254e8a633ff1dfeed7fd2daf80b78c4e8", + "name": "/lyakhonov-lab5-s3-1", + "started_at": "2026-11-20T03:39:52.148504843Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "654fa215c1b1d66c82b01a3c34e37e989671fdf4059f1b6a4fca3aada84c2311", + "name": "/lyakhonov-lab5-staging-cleaner-1", + "started_at": "2026-11-20T03:41:58.299334822Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66a77000f79c87a2e56fe362e46cf1a57bc2eb96cd8aa3eb60135f145655b1fa", + "name": "/lyakhonov-lab5-worker-1", + "started_at": "2026-11-20T03:56:40.43708125Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + }, + { + "id": "66e1949130cc05c7e83c8267a114318e9d53f5ff86d8b399be55e823b7597bfa", + "name": "/lyakhonov-lab5-worker-2", + "started_at": "2026-11-20T03:56:40.643246654Z", + "running": true, + "restart_count": 0, + "exit_code": 0 + } + ] + } +] \ No newline at end of file diff --git a/evidence/reproduction-run.txt b/evidence/reproduction-run.txt new file mode 100644 index 00000000..345f49da --- /dev/null +++ b/evidence/reproduction-run.txt @@ -0,0 +1 @@ +PASS: clean export, fresh secrets, isolated project; temporary Docker resources removed diff --git a/evidence/reproduction.json b/evidence/reproduction.json new file mode 100644 index 00000000..33b9ac76 --- /dev/null +++ b/evidence/reproduction.json @@ -0,0 +1,147 @@ +{ + "project": "lyakhonov-lab5-repro-ff60a514", + "directory": "C:\\Users\\User\\AppData\\Local\\Temp\\lyakhonov-lab5-repro-ff60a514-ia50tx4u", + "port": 58109, + "commit": "ef1f470659610f8ee0dc20bc3315589ba18154ca", + "events": [ + { + "command": [ + "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-X", + "utf8", + "scripts/bootstrap.py" + ], + "exit_code": 0, + "output": "Локальные файлы готовы\n" + }, + { + "command": [ + "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-X", + "utf8", + "scripts/check_contract.py" + ], + "exit_code": 0, + "output": "PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n" + }, + { + "command": [ + "docker", + "build", + "-t", + "lyakhonov-lab5-repro-ff60a514:app", + "." + ], + "exit_code": 0, + "output": "#0 building with \"desktop-linux\" instance using docker driver\n\n#1 [internal] load build definition from Dockerfile\n#1 transferring dockerfile: 793B 0.0s done\n#1 DONE 0.1s\n\n#2 [internal] load metadata for docker.io/library/python:3.11.10-slim-bookworm@sha256:840e180ebcc6e5c8efab209c43f5e40fd2af98cb49db5c7103c90539c56bb30e\n#2 DONE 1.1s\n\n#3 [internal] load .dockerignore\n#3 transferring context: 216B done\n#3 DONE 0.0s\n\n#4 [1/7] FROM docker.io/library/python:3.11.10-slim-bookworm@sha256:840e180ebcc6e5c8efab209c43f5e40fd2af98cb49db5c7103c90539c56bb30e\n#4 resolve docker.io/library/python:3.11.10-slim-bookworm@sha256:840e180ebcc6e5c8efab209c43f5e40fd2af98cb49db5c7103c90539c56bb30e 0.0s done\n#4 DONE 0.0s\n\n#5 [internal] load build context\n#5 transferring context: 84.25kB 0.0s done\n#5 DONE 0.0s\n\n#6 [2/7] WORKDIR /app\n#6 CACHED\n\n#7 [4/7] COPY requirements.lock.txt .\n#7 CACHED\n\n#8 [3/7] RUN groupadd -r appuser && useradd -r -g appuser appuser\n#8 CACHED\n\n#9 [5/7] RUN pip install --no-cache-dir -r requirements.lock.txt\n#9 CACHED\n\n#10 [6/7] COPY src/ src/\n#10 DONE 0.1s\n\n#11 [7/7] COPY migrations/ migrations/\n#11 DONE 0.0s\n\n#12 exporting to image\n#12 exporting layers 0.1s done\n#12 exporting manifest sha256:d7cdfb6d1c2c82e75c49b3d185a2ea6fcf6b5bbc0419385dc9a84981c8a7a758 0.0s done\n#12 exporting config sha256:6a6ea144015fac775c83db089852e96fff9323471e81c5502dce954beb9e2a59 0.0s done\n#12 exporting attestation manifest sha256:b03e62f4a5ade1f9e8d79126f57879bf371756f4f4b154afa530deac4b2954a6 0.0s done\n#12 exporting manifest list sha256:86b35a8e7434f91d2507907f772208fe8d121edc99152a1c58df058ebcc16a4b 0.0s done\n#12 naming to docker.io/library/lyakhonov-lab5-repro-ff60a514:app\n#12 naming to docker.io/library/lyakhonov-lab5-repro-ff60a514:app done\n#12 unpacking to docker.io/library/lyakhonov-lab5-repro-ff60a514:app 0.1s done\n#12 DONE 0.4s\n\nView build details: docker-desktop://dashboard/build/desktop-linux/desktop-linux/nuhn46pnzjnf324kzv2wyyshh\n" + }, + { + "command": [ + "docker", + "build", + "-t", + "lyakhonov-lab5-repro-ff60a514:s3", + "-f", + "Dockerfile.s3", + "." + ], + "exit_code": 0, + "output": "#0 building with \"desktop-linux\" instance using docker driver\n\n#1 [internal] load build definition from Dockerfile.s3\n#1 transferring dockerfile: 1.50kB 0.0s done\n#1 DONE 0.0s\n\n#2 [internal] load metadata for docker.io/library/golang:1.24.6-bookworm@sha256:ab1d1823abb55a9504d2e3e003b75b36dbeb1cbcc4c92593d85a84ee46becc6c\n#2 DONE 0.0s\n\n#3 [internal] load metadata for docker.io/library/debian:bookworm-20251020-slim@sha256:78d2f66e0fec9e5a39fb2c72ea5e052b548df75602b5215ed01a17171529f706\n#3 DONE 0.9s\n\n#4 [internal] load .dockerignore\n#4 transferring context: 216B done\n#4 DONE 0.0s\n\n#5 [build 6/7] ADD --checksum=sha256:526ec538e7f83e4bfa631788129b08ec85898ca572ce0523ebf9a597946bea27 https://github.com/minio/mc/archive/refs/tags/RELEASE.2025-04-16T18-13-26Z.tar.gz /tmp/mc.tar.gz\n#5 DONE 0.0s\n\n#6 [build 3/7] ADD --checksum=sha256:8819e3e7817e46b7b3798f8f200ead208562e571563c2e040352378031abe9f2 https://github.com/minio/minio/archive/07c3a429bfed433e49018cb0f78a52145d4bedeb.tar.gz /tmp/minio.tar.gz\n#6 DONE 0.0s\n\n#7 [internal] load build context\n#7 transferring context: 234B 0.0s done\n#7 DONE 0.0s\n\n#8 [stage-1 1/6] FROM docker.io/library/debian:bookworm-20251020-slim@sha256:78d2f66e0fec9e5a39fb2c72ea5e052b548df75602b5215ed01a17171529f706\n#8 resolve docker.io/library/debian:bookworm-20251020-slim@sha256:78d2f66e0fec9e5a39fb2c72ea5e052b548df75602b5215ed01a17171529f706 0.0s done\n#8 DONE 0.0s\n\n#9 [build 1/7] FROM docker.io/library/golang:1.24.6-bookworm@sha256:ab1d1823abb55a9504d2e3e003b75b36dbeb1cbcc4c92593d85a84ee46becc6c\n#9 resolve docker.io/library/golang:1.24.6-bookworm@sha256:ab1d1823abb55a9504d2e3e003b75b36dbeb1cbcc4c92593d85a84ee46becc6c 0.0s done\n#9 DONE 0.0s\n\n#10 [stage-1 3/6] COPY --from=build /build-mc/LICENSE /usr/share/doc/mc/LICENSE\n#10 CACHED\n\n#11 [stage-1 4/6] COPY --from=build /out/minio /out/mc /usr/bin/\n#11 CACHED\n\n#12 [stage-1 2/6] COPY --from=build /build/LICENSE /usr/share/doc/minio/LICENSE\n#12 CACHED\n\n#13 [build 4/7] RUN tar -xzf /tmp/minio.tar.gz --strip-components=1 && CGO_ENABLED=0 go build -trimpath -o /out/minio .\n#13 CACHED\n\n#14 [build 7/7] RUN tar -xzf /tmp/mc.tar.gz --strip-components=1 && CGO_ENABLED=0 go build -trimpath -o /out/mc .\n#14 CACHED\n\n#15 [build 3/7] ADD --checksum=sha256:8819e3e7817e46b7b3798f8f200ead208562e571563c2e040352378031abe9f2 https://github.com/minio/minio/archive/07c3a429bfed433e49018cb0f78a52145d4bedeb.tar.gz /tmp/minio.tar.gz\n#15 CACHED\n\n#16 [build 6/7] ADD --checksum=sha256:526ec538e7f83e4bfa631788129b08ec85898ca572ce0523ebf9a597946bea27 https://github.com/minio/mc/archive/refs/tags/RELEASE.2025-04-16T18-13-26Z.tar.gz /tmp/mc.tar.gz\n#16 CACHED\n\n#17 [build 2/7] WORKDIR /build\n#17 CACHED\n\n#18 [build 5/7] WORKDIR /build-mc\n#18 CACHED\n\n#19 [stage-1 5/6] COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt\n#19 CACHED\n\n#20 [stage-1 6/6] COPY infra/s3-entrypoint.sh /usr/local/bin/s3-entrypoint.sh\n#20 CACHED\n\n#21 exporting to image\n#21 exporting layers done\n#21 exporting manifest sha256:716ae1c0f3870057746078ed9135e97d8864dc3d5888d65a5262b3201cf92aec 0.0s done\n#21 exporting config sha256:ac09ab12a3c4db268f02f1597be22a25f4d4c0541f4fd7d834af7457f3fbeb96\n#21 exporting config sha256:ac09ab12a3c4db268f02f1597be22a25f4d4c0541f4fd7d834af7457f3fbeb96 0.0s done\n#21 exporting attestation manifest sha256:bcebdec6c842af8c8554942426fb730d0631e7317f88c192009218bd353057eb 0.0s done\n#21 exporting manifest list sha256:2d19b26306cc58385ab9081032e398f7c12ef0a753878d73114a00ac104171d8 done\n#21 naming to docker.io/library/lyakhonov-lab5-repro-ff60a514:s3 done\n#21 unpacking to docker.io/library/lyakhonov-lab5-repro-ff60a514:s3 done\n#21 DONE 0.1s\n\nView build details: docker-desktop://dashboard/build/desktop-linux/desktop-linux/04xxhhn255nn5pvdf0ugdahmx\n" + }, + { + "command": [ + "docker", + "compose", + "-p", + "lyakhonov-lab5-repro-ff60a514", + "-f", + "compose.yaml", + "-f", + "repro-override.json", + "up", + "-d" + ], + "exit_code": 0, + "output": " Network lyakhonov-lab5-repro-ff60a514_default Creating\n Network lyakhonov-lab5-repro-ff60a514_default Created\n Volume lyakhonov-lab5-repro-ff60a514_s3_data Creating\n Volume lyakhonov-lab5-repro-ff60a514_s3_data Created\n Volume lyakhonov-lab5-repro-ff60a514_postgres_data Creating\n Volume lyakhonov-lab5-repro-ff60a514_postgres_data Created\n Volume lyakhonov-lab5-repro-ff60a514_broker_data Creating\n Volume lyakhonov-lab5-repro-ff60a514_broker_data Created\n Container lyakhonov-lab5-repro-ff60a514-broker-1 Creating\n Container lyakhonov-lab5-repro-ff60a514-redis-1 Creating\n Container lyakhonov-lab5-repro-ff60a514-s3-1 Creating\n Container lyakhonov-lab5-repro-ff60a514-db-1 Creating\n Container lyakhonov-lab5-repro-ff60a514-redis-1 Created\n Container lyakhonov-lab5-repro-ff60a514-broker-1 Created\n Container lyakhonov-lab5-repro-ff60a514-db-1 Created\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Creating\n Container lyakhonov-lab5-repro-ff60a514-s3-1 Created\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Creating\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Created\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Created\n Container lyakhonov-lab5-repro-ff60a514-dispatcher-1 Creating\n Container lyakhonov-lab5-repro-ff60a514-api-2 Creating\n Container lyakhonov-lab5-repro-ff60a514-api-1 Creating\n Container lyakhonov-lab5-repro-ff60a514-worker-2 Creating\n Container lyakhonov-lab5-repro-ff60a514-worker-1 Creating\n Container lyakhonov-lab5-repro-ff60a514-staging-cleaner-1 Creating\n Container lyakhonov-lab5-repro-ff60a514-worker-1 Created\n Container lyakhonov-lab5-repro-ff60a514-api-2 Created\n Container lyakhonov-lab5-repro-ff60a514-worker-2 Created\n Container lyakhonov-lab5-repro-ff60a514-dispatcher-1 Created\n Container lyakhonov-lab5-repro-ff60a514-staging-cleaner-1 Created\n Container lyakhonov-lab5-repro-ff60a514-api-1 Created\n Container lyakhonov-lab5-repro-ff60a514-lb-1 Creating\n Container lyakhonov-lab5-repro-ff60a514-lb-1 Created\n Container lyakhonov-lab5-repro-ff60a514-db-1 Starting\n Container lyakhonov-lab5-repro-ff60a514-redis-1 Starting\n Container lyakhonov-lab5-repro-ff60a514-broker-1 Starting\n Container lyakhonov-lab5-repro-ff60a514-s3-1 Starting\n Container lyakhonov-lab5-repro-ff60a514-s3-1 Started\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Starting\n Container lyakhonov-lab5-repro-ff60a514-broker-1 Started\n Container lyakhonov-lab5-repro-ff60a514-db-1 Started\n Container lyakhonov-lab5-repro-ff60a514-db-1 Waiting\n Container lyakhonov-lab5-repro-ff60a514-redis-1 Started\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Started\n Container lyakhonov-lab5-repro-ff60a514-db-1 Healthy\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Starting\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Started\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Waiting\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Waiting\n Container lyakhonov-lab5-repro-ff60a514-broker-1 Waiting\n Container lyakhonov-lab5-repro-ff60a514-broker-1 Waiting\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Waiting\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Waiting\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Waiting\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Waiting\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Waiting\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Waiting\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Exited\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Exited\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Exited\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Exited\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Exited\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Exited\n Container lyakhonov-lab5-repro-ff60a514-api-1 Starting\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Exited\n Container lyakhonov-lab5-repro-ff60a514-staging-cleaner-1 Starting\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Exited\n Container lyakhonov-lab5-repro-ff60a514-api-1 Started\n Container lyakhonov-lab5-repro-ff60a514-api-2 Starting\n Container lyakhonov-lab5-repro-ff60a514-staging-cleaner-1 Started\n Container lyakhonov-lab5-repro-ff60a514-api-2 Started\n Container lyakhonov-lab5-repro-ff60a514-lb-1 Starting\n Container lyakhonov-lab5-repro-ff60a514-lb-1 Started\n Container lyakhonov-lab5-repro-ff60a514-broker-1 Healthy\n Container lyakhonov-lab5-repro-ff60a514-broker-1 Healthy\n Container lyakhonov-lab5-repro-ff60a514-worker-2 Starting\n Container lyakhonov-lab5-repro-ff60a514-dispatcher-1 Starting\n Container lyakhonov-lab5-repro-ff60a514-worker-2 Started\n Container lyakhonov-lab5-repro-ff60a514-worker-1 Starting\n Container lyakhonov-lab5-repro-ff60a514-dispatcher-1 Started\n Container lyakhonov-lab5-repro-ff60a514-worker-1 Started\n" + }, + { + "command": [ + "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-X", + "utf8", + "scripts/wait_ready.py", + "--project", + "lyakhonov-lab5-repro-ff60a514", + "--base-url", + "http://127.0.0.1:58109" + ], + "exit_code": 0, + "output": "PASS: two ready API instances, healthy DB/broker, required background containers running\n" + }, + { + "command": [ + "C:\\Users\\User\\AppData\\Local\\Programs\\Python\\Python313\\python.exe", + "-X", + "utf8", + "tests/smoke.py", + "--base-url", + "http://127.0.0.1:58109" + ], + "exit_code": 0, + "output": "PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.\n" + }, + { + "command": [ + "docker", + "compose", + "-p", + "lyakhonov-lab5-repro-ff60a514", + "-f", + "compose.yaml", + "-f", + "repro-override.json", + "ps" + ], + "exit_code": 0, + "output": "NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS\nlyakhonov-lab5-repro-ff60a514-api-1 lyakhonov-lab5-repro-ff60a514:app \"python -m uvicorn s…\" api 24 seconds ago Up 14 seconds (healthy) 8080/tcp\nlyakhonov-lab5-repro-ff60a514-api-2 lyakhonov-lab5-repro-ff60a514:app \"python -m uvicorn s…\" api 24 seconds ago Up 14 seconds (healthy) 8080/tcp\nlyakhonov-lab5-repro-ff60a514-broker-1 rabbitmq:4.1.0-alpine \"/bin/sh /bootstrap/…\" broker 25 seconds ago Up 22 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp\nlyakhonov-lab5-repro-ff60a514-db-1 postgres:16.6-alpine \"docker-entrypoint.s…\" db 25 seconds ago Up 22 seconds (healthy) 5432/tcp\nlyakhonov-lab5-repro-ff60a514-dispatcher-1 lyakhonov-lab5-repro-ff60a514:app \"python -m src.dispa…\" dispatcher 24 seconds ago Up 5 seconds 8080/tcp\nlyakhonov-lab5-repro-ff60a514-lb-1 haproxy:3.0.6-alpine \"docker-entrypoint.s…\" lb 24 seconds ago Up 14 seconds 127.0.0.1:58109->8080/tcp\nlyakhonov-lab5-repro-ff60a514-redis-1 redis:7.4.1-alpine \"docker-entrypoint.s…\" redis 25 seconds ago Up 22 seconds 6379/tcp\nlyakhonov-lab5-repro-ff60a514-s3-1 lyakhonov-lab5-repro-ff60a514:s3 \"/bin/sh /usr/local/…\" s3 25 seconds ago Up 22 seconds \nlyakhonov-lab5-repro-ff60a514-staging-cleaner-1 lyakhonov-lab5-repro-ff60a514:app \"python -m src.stagi…\" staging-cleaner 24 seconds ago Up 14 seconds 8080/tcp\nlyakhonov-lab5-repro-ff60a514-worker-1 lyakhonov-lab5-repro-ff60a514:app \"python -m src.worker\" worker 24 seconds ago Up 4 seconds 8080/tcp\nlyakhonov-lab5-repro-ff60a514-worker-2 lyakhonov-lab5-repro-ff60a514:app \"python -m src.worker\" worker 24 seconds ago Up 5 seconds 8080/tcp\n" + }, + { + "command": [ + "docker", + "compose", + "-p", + "lyakhonov-lab5-repro-ff60a514", + "-f", + "compose.yaml", + "-f", + "repro-override.json", + "down", + "--volumes", + "--remove-orphans" + ], + "exit_code": 0, + "output": " Container lyakhonov-lab5-repro-ff60a514-redis-1 Stopping\n Container lyakhonov-lab5-repro-ff60a514-dispatcher-1 Stopping\n Container lyakhonov-lab5-repro-ff60a514-worker-1 Stopping\n Container lyakhonov-lab5-repro-ff60a514-staging-cleaner-1 Stopping\n Container lyakhonov-lab5-repro-ff60a514-lb-1 Stopping\n Container lyakhonov-lab5-repro-ff60a514-worker-2 Stopping\n Container lyakhonov-lab5-repro-ff60a514-lb-1 Stopped\n Container lyakhonov-lab5-repro-ff60a514-lb-1 Removing\n Container lyakhonov-lab5-repro-ff60a514-lb-1 Removed\n Container lyakhonov-lab5-repro-ff60a514-api-2 Stopping\n Container lyakhonov-lab5-repro-ff60a514-api-1 Stopping\n Container lyakhonov-lab5-repro-ff60a514-redis-1 Stopped\n Container lyakhonov-lab5-repro-ff60a514-redis-1 Removing\n Container lyakhonov-lab5-repro-ff60a514-redis-1 Removed\n Container lyakhonov-lab5-repro-ff60a514-worker-1 Stopped\n Container lyakhonov-lab5-repro-ff60a514-worker-1 Removing\n Container lyakhonov-lab5-repro-ff60a514-worker-1 Removed\n Container lyakhonov-lab5-repro-ff60a514-dispatcher-1 Stopped\n Container lyakhonov-lab5-repro-ff60a514-dispatcher-1 Removing\n Container lyakhonov-lab5-repro-ff60a514-dispatcher-1 Removed\n Container lyakhonov-lab5-repro-ff60a514-worker-2 Stopped\n Container lyakhonov-lab5-repro-ff60a514-worker-2 Removing\n Container lyakhonov-lab5-repro-ff60a514-worker-2 Removed\n Container lyakhonov-lab5-repro-ff60a514-broker-1 Stopping\n Container lyakhonov-lab5-repro-ff60a514-staging-cleaner-1 Stopped\n Container lyakhonov-lab5-repro-ff60a514-staging-cleaner-1 Removing\n Container lyakhonov-lab5-repro-ff60a514-staging-cleaner-1 Removed\n Container lyakhonov-lab5-repro-ff60a514-api-2 Stopped\n Container lyakhonov-lab5-repro-ff60a514-api-2 Removing\n Container lyakhonov-lab5-repro-ff60a514-api-2 Removed\n Container lyakhonov-lab5-repro-ff60a514-api-1 Stopped\n Container lyakhonov-lab5-repro-ff60a514-api-1 Removing\n Container lyakhonov-lab5-repro-ff60a514-api-1 Removed\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Stopping\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Stopping\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Stopped\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Removing\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Stopped\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Removing\n Container lyakhonov-lab5-repro-ff60a514-migrate-1 Removed\n Container lyakhonov-lab5-repro-ff60a514-db-1 Stopping\n Container lyakhonov-lab5-repro-ff60a514-bucket-init-1 Removed\n Container lyakhonov-lab5-repro-ff60a514-s3-1 Stopping\n Container lyakhonov-lab5-repro-ff60a514-broker-1 Stopped\n Container lyakhonov-lab5-repro-ff60a514-broker-1 Removing\n Container lyakhonov-lab5-repro-ff60a514-broker-1 Removed\n Container lyakhonov-lab5-repro-ff60a514-s3-1 Stopped\n Container lyakhonov-lab5-repro-ff60a514-s3-1 Removing\n Container lyakhonov-lab5-repro-ff60a514-db-1 Stopped\n Container lyakhonov-lab5-repro-ff60a514-db-1 Removing\n Container lyakhonov-lab5-repro-ff60a514-s3-1 Removed\n Container lyakhonov-lab5-repro-ff60a514-db-1 Removed\n Volume lyakhonov-lab5-repro-ff60a514_broker_data Removing\n Volume lyakhonov-lab5-repro-ff60a514_s3_data Removing\n Network lyakhonov-lab5-repro-ff60a514_default Removing\n Volume lyakhonov-lab5-repro-ff60a514_postgres_data Removing\n Volume lyakhonov-lab5-repro-ff60a514_broker_data Removed\n Volume lyakhonov-lab5-repro-ff60a514_s3_data Removed\n Volume lyakhonov-lab5-repro-ff60a514_postgres_data Removed\n Network lyakhonov-lab5-repro-ff60a514_default Removed\n" + }, + { + "command": [ + "docker", + "image", + "rm", + "lyakhonov-lab5-repro-ff60a514:app" + ], + "exit_code": 0, + "output": "Untagged: lyakhonov-lab5-repro-ff60a514:app\nDeleted: sha256:86b35a8e7434f91d2507907f772208fe8d121edc99152a1c58df058ebcc16a4b\n" + }, + { + "command": [ + "docker", + "image", + "rm", + "lyakhonov-lab5-repro-ff60a514:s3" + ], + "exit_code": 0, + "output": "Untagged: lyakhonov-lab5-repro-ff60a514:s3\nDeleted: sha256:2d19b26306cc58385ab9081032e398f7c12ef0a753878d73114a00ac104171d8\n" + } + ], + "temporary_directory_removed": true +} \ No newline at end of file diff --git a/evidence/resilience-initial-run.txt b/evidence/resilience-initial-run.txt new file mode 100644 index 00000000..27595d61 --- /dev/null +++ b/evidence/resilience-initial-run.txt @@ -0,0 +1,37 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Starting + Container lyakhonov-lab5-bucket-init-1 Started + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Starting + Container lyakhonov-lab5-migrate-1 Started + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 88, "failed_attempts": 4, "first_recovery_seconds": 2.856} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + + Container lyakhonov-lab5-bucket-init-1 Starting + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Started + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Starting + Container lyakhonov-lab5-migrate-1 Started + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-api-2 Starting + Container lyakhonov-lab5-api-2 Started +Traceback (most recent call last): + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\resilience_driver.py", line 82, in + targeted() + ~~~~~~~~^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\resilience_driver.py", line 62, in targeted + messages = list(page['items']) + ~~~~^^^^^^^^^ +TypeError: 'NoneType' object is not subscriptable diff --git a/evidence/resilience-run.txt b/evidence/resilience-run.txt new file mode 100644 index 00000000..27595d61 --- /dev/null +++ b/evidence/resilience-run.txt @@ -0,0 +1,37 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Starting + Container lyakhonov-lab5-bucket-init-1 Started + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Starting + Container lyakhonov-lab5-migrate-1 Started + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 88, "failed_attempts": 4, "first_recovery_seconds": 2.856} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + + Container lyakhonov-lab5-bucket-init-1 Starting + Container lyakhonov-lab5-db-1 Waiting + Container lyakhonov-lab5-bucket-init-1 Started + Container lyakhonov-lab5-db-1 Healthy + Container lyakhonov-lab5-migrate-1 Starting + Container lyakhonov-lab5-migrate-1 Started + Container lyakhonov-lab5-bucket-init-1 Waiting + Container lyakhonov-lab5-migrate-1 Waiting + Container lyakhonov-lab5-migrate-1 Exited + Container lyakhonov-lab5-bucket-init-1 Exited + Container lyakhonov-lab5-api-2 Starting + Container lyakhonov-lab5-api-2 Started +Traceback (most recent call last): + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\resilience_driver.py", line 82, in + targeted() + ~~~~~~~~^^ + File "C:\Users\User\Desktop\vysoko-nagr 5\student-tests\resilience_driver.py", line 62, in targeted + messages = list(page['items']) + ~~~~^^^^^^^^^ +TypeError: 'NoneType' object is not subscriptable diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..5c90482b --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/structure.txt b/evidence/structure.txt new file mode 100644 index 00000000..b62af9c5 --- /dev/null +++ b/evidence/structure.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/unit.txt b/evidence/unit.txt new file mode 100644 index 00000000..3e7cac78 --- /dev/null +++ b/evidence/unit.txt @@ -0,0 +1,5 @@ +.. +---------------------------------------------------------------------- +Ran 2 tests in 0.142s + +OK diff --git a/evidence/validation.txt b/evidence/validation.txt new file mode 100644 index 00000000..248343ee --- /dev/null +++ b/evidence/validation.txt @@ -0,0 +1 @@ +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/infra/broker.sh b/infra/broker.sh new file mode 100644 index 00000000..bb064230 --- /dev/null +++ b/infra/broker.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +export RABBITMQ_DEFAULT_USER=messenger +export RABBITMQ_DEFAULT_PASS="$(cat /run/secrets/broker_password)" +exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server diff --git a/infra/bucket.sh b/infra/bucket.sh new file mode 100644 index 00000000..acf28ff1 --- /dev/null +++ b/infra/bucket.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +set -eu + +# Не включать set -x: аргументы содержат учебные credentials. + +attempt=0 +until mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 30 ]; then + echo 'S3 bootstrap timed out' >&2 + exit 1 + fi + sleep 1 +done + +mc mb --ignore-existing local/attachments >/dev/null + +mc anonymous set none local/attachments >/dev/null + +mc admin user add local "$(cat /run/secrets/s3_access_key)" "$(cat /run/secrets/s3_secret_key)" >/dev/null + +mc admin policy create local attachments-app /bootstrap/policy.json >/dev/null + +mc admin policy attach local attachments-app --user "$(cat /run/secrets/s3_access_key)" >/dev/null + +echo 'Private bucket and restricted application user ready' + diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..042a6cb2 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,27 @@ +global + log stdout format raw local0 + maxconn 1024 +defaults + mode http + log global + timeout connect 1s + timeout client 60s + timeout server 45s + timeout check 1s + retries 0 +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s +frontend public + bind :8080 + default_backend application +backend application + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 1 + server-template api 1-4 api:8080 check resolvers docker init-addr none diff --git a/infra/rabbitmq.conf b/infra/rabbitmq.conf new file mode 100644 index 00000000..c150c13a --- /dev/null +++ b/infra/rabbitmq.conf @@ -0,0 +1,3 @@ +heartbeat = 180 +vm_memory_high_watermark.absolute = 300MiB +disk_free_limit.absolute = 200MiB diff --git a/infra/s3-entrypoint.sh b/infra/s3-entrypoint.sh new file mode 100644 index 00000000..2134296e --- /dev/null +++ b/infra/s3-entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +export MINIO_ROOT_USER="$(cat "$MINIO_ROOT_USER_FILE")" +export MINIO_ROOT_PASSWORD="$(cat "$MINIO_ROOT_PASSWORD_FILE")" +exec minio "$@" diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 00000000..1bff5854 --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1 @@ +{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::attachments"],"Condition":{"StringLike":{"s3:prefix":["staging/*"]}}},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::attachments/originals/*","arn:aws:s3:::attachments/variants/*"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::attachments/staging/*"]}]} diff --git a/lab5.env.example b/lab5.env.example new file mode 100644 index 00000000..7096d481 --- /dev/null +++ b/lab5.env.example @@ -0,0 +1,7 @@ +LAB5_PORT=8085 +API_REPLICAS=2 +SESSION_TTL_SECONDS=3600 +WORKER_CONCURRENCY=2 +JOB_MAX_ATTEMPTS=5 +JOB_TIMEOUT_SECONDS=60 +STAGING_RETENTION_SECONDS=86400 diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..2ee14af7 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,33 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name varchar(100) NOT NULL CHECK (length(btrim(display_name)) > 0), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL CHECK (length(btrim(title)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text varchar(4000) NOT NULL CHECK (length(btrim(text)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); +CREATE TABLE sessions ( + token_hash char(64) PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..be707c47 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE message_requests ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key varchar(128) NOT NULL, + body text NOT NULL, + message_id uuid NOT NULL REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, key) +); +CREATE INDEX message_requests_expiry_idx ON message_requests(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..9bdc747c --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,17 @@ +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_limit CHECK (length(text)<=4000); +CREATE TABLE attachments ( + id uuid PRIMARY KEY, chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), object_key text NOT NULL UNIQUE, + filename text NOT NULL, content_type text NOT NULL, size_bytes integer NOT NULL, + sha256 char(64) NOT NULL, state text NOT NULL CHECK(state IN ('ready','missing')), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY(chat_id,owner_id) REFERENCES chat_members(chat_id,user_id) +); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id), attachment_id uuid NOT NULL REFERENCES attachments(id), + position integer NOT NULL, PRIMARY KEY(message_id,attachment_id), UNIQUE(message_id,position) +); +-- Старые сохранённые ключи теперь сравнивают полное нормализованное тело. +UPDATE message_requests SET body=json_build_object('text',body,'attachment_ids',json_build_array())::text; diff --git a/migrations/004_attachment_pipeline.sql b/migrations/004_attachment_pipeline.sql new file mode 100644 index 00000000..2e2eb707 --- /dev/null +++ b/migrations/004_attachment_pipeline.sql @@ -0,0 +1,49 @@ +ALTER TABLE attachments DROP CONSTRAINT attachments_state_check; +UPDATE attachments SET state = 'failed' WHERE state = 'missing'; +ALTER TABLE attachments ADD COLUMN error_code text; +UPDATE attachments SET error_code = 'original_unavailable' WHERE state = 'failed'; +ALTER TABLE attachments ADD CONSTRAINT attachments_pipeline_state CHECK ( + state IN ('queued', 'processing', 'ready', 'failed') + AND ((state = 'failed') = (error_code IS NOT NULL)) +); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id), + staging_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + attempt_limit integer NOT NULL CHECK (attempt_limit >= 1), + state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'processing', 'ready', 'failed')), + next_publish_at timestamptz, + published_at timestamptz, + publish_failures integer NOT NULL DEFAULT 0, + lease_until timestamptz, + lease_token uuid, + last_error text, + CHECK ((lease_until IS NULL) = (lease_token IS NULL)) +); +CREATE INDEX attachment_jobs_outbox ON attachment_jobs(next_publish_at) + WHERE next_publish_at IS NOT NULL; +CREATE INDEX attachment_jobs_leases ON attachment_jobs(lease_until) + WHERE lease_until IS NOT NULL; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id), + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL, + size_bytes integer NOT NULL CHECK (size_bytes > 0), + sha256 char(64) NOT NULL, + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + PRIMARY KEY (attachment_id, name) +); + +CREATE TABLE attachment_replays ( + job_id uuid NOT NULL REFERENCES attachment_jobs(id), + attempt integer NOT NULL, + reason text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (job_id, attempt) +); diff --git a/migrations/005_versioned_pipeline.sql b/migrations/005_versioned_pipeline.sql new file mode 100644 index 00000000..760c52b7 --- /dev/null +++ b/migrations/005_versioned_pipeline.sql @@ -0,0 +1,19 @@ +ALTER TABLE attachments + ADD COLUMN target_pipeline_version integer NOT NULL DEFAULT 1 CHECK (target_pipeline_version >= 1), + ADD COLUMN published_pipeline_version integer CHECK (published_pipeline_version >= 1), + ADD CHECK (published_pipeline_version <= target_pipeline_version); +UPDATE attachments SET published_pipeline_version = 1 WHERE state = 'ready'; + +ALTER TABLE attachment_jobs + DROP CONSTRAINT attachment_jobs_attachment_id_key, + DROP CONSTRAINT attachment_jobs_staging_key_key, + DROP CONSTRAINT attachment_jobs_state_check, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1), + ADD COLUMN settled_at timestamptz, + ADD CHECK (state IN ('queued', 'processing', 'ready', 'failed', 'superseded')), + ADD UNIQUE (attachment_id, pipeline_version); +CREATE INDEX attachment_jobs_staging ON attachment_jobs(staging_key); + +ALTER TABLE attachment_variants ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; +ALTER TABLE attachment_variants DROP CONSTRAINT attachment_variants_pkey; +ALTER TABLE attachment_variants ADD PRIMARY KEY (attachment_id, pipeline_version, name); diff --git a/migrations/006_experiment_faults.sql b/migrations/006_experiment_faults.sql new file mode 100644 index 00000000..3a5e556e --- /dev/null +++ b/migrations/006_experiment_faults.sql @@ -0,0 +1,9 @@ +CREATE TABLE experiment_faults ( + job_id uuid NOT NULL REFERENCES attachment_jobs(id), + point text NOT NULL, + action text NOT NULL CHECK (action IN ('crash', 'pause')), + hit_at timestamptz, + process_id integer, + released boolean NOT NULL DEFAULT false, + PRIMARY KEY (job_id, point) +); diff --git a/requirements-research.txt b/requirements-research.txt new file mode 100644 index 00000000..d8323040 --- /dev/null +++ b/requirements-research.txt @@ -0,0 +1,2 @@ +-r requirements-dev.txt +matplotlib==3.10.7 diff --git a/requirements.lock.txt b/requirements.lock.txt new file mode 100644 index 00000000..4da3e7c2 --- /dev/null +++ b/requirements.lock.txt @@ -0,0 +1,35 @@ +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==23.1.0 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +fastapi==0.115.0 +h11==0.16.0 +httptools==0.8.0 +idna==3.20 +psycopg==3.2.3 +psycopg-binary==3.2.3 +psycopg-pool==3.2.4 +pycparser==3.1 +pydantic==2.9.2 +pydantic_core==2.23.4 +python-dotenv==1.2.4 +PyYAML==6.0.3 +starlette==0.38.6 +typing_extensions==4.16.0 +uvicorn==0.32.0 +uvloop==0.23.0 +watchfiles==1.3.0 +websockets==17.2 +redis==5.2.0 + +boto3==1.40.61 +Pillow==12.0.0 +pika==1.3.2 +botocore==1.40.76 +s3transfer==0.14.0 +jmespath==1.1.0 +python-dateutil==2.9.0.post0 +six==1.17.0 +urllib3==2.8.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..d27c7f25 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +fastapi==0.115.0 +uvicorn[standard]==0.32.0 +pydantic==2.9.2 +psycopg[binary]==3.2.3 +psycopg-pool==3.2.4 +argon2-cffi==23.1.0 +redis==5.2.0 + +boto3==1.40.61 +Pillow==12.0.0 +pika==1.3.2 diff --git a/scripts/actions.py b/scripts/actions.py new file mode 100644 index 00000000..48246fee --- /dev/null +++ b/scripts/actions.py @@ -0,0 +1,31 @@ +"""Windows hooks for the public resilience client.""" + +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def docker(*args): + return subprocess.check_output(['docker', *args], cwd=ROOT, text=True).strip() + + +def main(): + if sys.argv[1] == 'restart': + docker('compose', 'restart', '-t', '4', 'db', 'redis', 's3', 'broker', 'api', + 'dispatcher', 'worker', 'staging-cleaner', 'lb') + return + target = os.environ['TARGET_INSTANCE_ID'] + for container in docker('compose', 'ps', '-q', 'api').splitlines(): + name = docker('inspect', '--format', '{{.Config.Hostname}}', container) + prefix, _, suffix = target.rpartition('-') + if prefix == name and len(suffix) == 8: + docker('stop', '-t', '2', container) + return + raise SystemExit('API instance not found') + + +if __name__ == '__main__': + main() diff --git a/scripts/audit.py b/scripts/audit.py new file mode 100644 index 00000000..cfc4fc8c --- /dev/null +++ b/scripts/audit.py @@ -0,0 +1,55 @@ +"""Аудит обязательных материалов, локальных секретов и предыдущей лабораторной.""" +import hashlib +import json +import os +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +PREVIOUS = ROOT.parent / 'vysoko-nagr 4' +ENV = dict(os.environ, GIT_OPTIONAL_LOCKS='0') + + +def git(root, *args): + return subprocess.check_output(['git', '-C', str(root), *args], env=ENV).decode('utf-8').strip() + + +def previous(): + paths = git(PREVIOUS, 'ls-files', '-z').split('\0') + files = {p: hashlib.sha256((PREVIOUS / p).read_bytes()).hexdigest() for p in paths if p} + return dict(head=git(PREVIOUS, 'rev-parse', 'HEAD'), branch=git(PREVIOUS, 'branch', '--show-current'), + status=git(PREVIOUS, 'status', '--porcelain'), files=files) + + +def main(): + import sys + if sys.argv[1:] == ['baseline']: + (ROOT / 'evidence/lab4-baseline.json').write_text(json.dumps(previous(), ensure_ascii=False, indent=2), encoding='utf-8') + print('Сохранён read-only fingerprint лабораторной 4') + return + baseline = json.loads((ROOT / 'evidence/lab4-baseline.json').read_text(encoding='utf-8')) + unchanged = previous() == baseline + assert unchanged, 'Lab4 differs from baseline' + protected = ['README.md', 'COURSE.md', 'CONTRIBUTING.md', 'MAINTAINERS.md', 'contracts', 'tests', + 'examples', 'compose.example.yaml', '.env.example', '.github'] + assert not git(ROOT, 'diff', '276dbda', '--', *protected), 'Protected lab5 material changed' + paths = git(ROOT, 'ls-files', '-z').split('\0') + secret_values = [p.read_bytes().strip() for p in (ROOT / 'secrets').iterdir() if p.is_file()] + for path in paths: + if not path: + continue + assert not path.startswith('secrets/') and path != '.env', 'Tracked private state' + content = (ROOT / path).read_bytes() + assert not any(value and value in content for value in secret_values), 'Local secret leaked in ' + path + assert (b'-----BEGIN ' + b'PRIVATE KEY-----') not in content, 'Private key found in ' + path + authors = git(ROOT, 'log', '276dbda..HEAD', '--format=%h|%an|%ae|%cn|%ce').splitlines() + for row in authors: + assert row.split('|')[1:] == ['Sergey Lyakhonov', 'lyserg119@gmail.com'] * 2, 'Wrong author/committer' + report = dict(lab4_unchanged=unchanged, protected_lab5_unchanged=True, tracked_local_secrets=False, + scanned_files=len(paths), authors=authors, head=git(ROOT, 'rev-parse', 'HEAD')) + (ROOT / 'evidence/audit.json').write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding='utf-8') + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..1b4cc364 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,33 @@ +"""Создаёт только отсутствующие локальные секреты и конфигурацию.""" + +from pathlib import Path +from secrets import token_hex +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] + + +def create(path, value): + try: + with path.open('x', encoding='utf-8') as target: + target.write(value) + except FileExistsError: + pass + + +def main(): + directory = ROOT / 'secrets' + directory.mkdir(mode=0o700, exist_ok=True) + for name in ('db_password', 'broker_password', 's3_root_user', 's3_root_password', + 's3_access_key', 's3_secret_key'): + create(directory / name, token_hex(24)) + db = quote((directory / 'db_password').read_text(encoding='utf-8').strip(), safe='') + broker = quote((directory / 'broker_password').read_text(encoding='utf-8').strip(), safe='') + create(directory / 'database_url', f'postgresql://messenger:{db}@db:5432/messenger') + create(directory / 'broker_url', f'amqp://messenger:{broker}@broker:5672/%2F') + create(ROOT / '.env', (ROOT / 'lab5.env.example').read_text(encoding='utf-8')) + print('Локальные файлы готовы') + + +if __name__ == '__main__': + main() diff --git a/scripts/environment.py b/scripts/environment.py new file mode 100644 index 00000000..f45db923 --- /dev/null +++ b/scripts/environment.py @@ -0,0 +1,30 @@ +"""Безопасная фиксация версий, лимитов и опубликованных портов.""" +import json +from pathlib import Path +import platform +import subprocess + +ROOT = Path(__file__).resolve().parents[1] + + +def output(*args): + return subprocess.check_output(args, cwd=ROOT, text=True, encoding='utf-8').strip() + + +ids = output('docker', 'compose', '-p', 'lyakhonov-lab5', 'ps', '-q').split() +containers = json.loads(output('docker', 'inspect', *ids)) +result = dict(commit=output('git', 'rev-parse', 'HEAD'), python=platform.python_version(), + working_tree_dirty=bool(output('git', 'status', '--porcelain')), + docker=json.loads(output('docker', 'version', '--format', '{{json .}}')), + compose=output('docker', 'compose', 'version'), + engine=json.loads(output('docker', 'info', '--format', '{"memory_bytes":{{.MemTotal}},"cpus":{{.NCPU}}}')), + services=[]) +for row in containers: + result['services'].append(dict(service=row['Config']['Labels']['com.docker.compose.service'], id=row['Id'], + image=row['Config']['Image'], image_id=row['Image'], started_at=row['State']['StartedAt'], + restart_count=row['RestartCount'], oom_killed=row['State'].get('OOMKilled', False), + memory_limit=row['HostConfig']['Memory'], nanocpus=row['HostConfig']['NanoCpus'], + ports=row['HostConfig']['PortBindings'], + settings=[x for x in row['Config']['Env'] if x.startswith(('JOB_', 'WORKER_', 'LAB5_FAULT_', 'STAGING_RETENTION_'))])) +(ROOT / 'evidence/environment.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8') +print('Saved versions, IDs, resource limits, ports and non-secret experiment settings') diff --git a/scripts/final_state.py b/scripts/final_state.py new file mode 100644 index 00000000..5555d27b --- /dev/null +++ b/scripts/final_state.py @@ -0,0 +1,36 @@ +"""Финальные агрегаты собственного стенда без пользовательских данных и секретов.""" +import json +from pathlib import Path +import subprocess +import re + +ROOT = Path(__file__).resolve().parents[1] +load = json.loads((ROOT / 'evidence/load-pipeline.json').read_text(encoding='utf-8')) +failed_ids = sorted(set(re.findall(r'[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}', + ' '.join(error for row in load for error in row['failures'])))) +script = 'FAILED_IDS=' + repr(failed_ids) + '\n' + ''' +import json +from src.storage import storage +storage.pool.open(wait=True) +try: + with storage.pool.connection() as conn: + result = { + 'attachments': conn.execute('SELECT state,count(*) AS count FROM attachments GROUP BY state').fetchall(), + 'jobs': conn.execute('SELECT state,pipeline_version,count(*) AS count FROM attachment_jobs GROUP BY state,pipeline_version ORDER BY pipeline_version,state').fetchall(), + 'replays': conn.execute('SELECT count(*) AS count FROM attachment_replays').fetchone()['count'], + 'load_observation_errors': conn.execute('SELECT id,state,published_pipeline_version,target_pipeline_version FROM attachments WHERE id=ANY(%s::uuid[])', (FAILED_IDS,)).fetchall(), + 'violations': conn.execute("""SELECT count(*) AS count FROM attachments a + WHERE state <> 'ready' OR published_pipeline_version <> target_pipeline_version + OR (SELECT count(*) FROM attachment_variants v WHERE v.attachment_id=a.id + AND v.pipeline_version=a.published_pipeline_version) <> 3""").fetchone()['count'] + } + print(json.dumps(result, default=str)) +finally: + storage.pool.close() +''' +output = subprocess.check_output(['docker','compose','-p','lyakhonov-lab5','exec','-T','--index','1','api','python','-'], + input=script, cwd=ROOT, text=True, encoding='utf-8') +result = json.loads(output) +(ROOT / 'evidence/final-state.json').write_text(json.dumps(result, indent=2), encoding='utf-8') +assert result['violations'] == 0, 'Incomplete or inconsistent current manifests' +print(output) diff --git a/scripts/plot_results.py b/scripts/plot_results.py new file mode 100644 index 00000000..b0621deb --- /dev/null +++ b/scripts/plot_results.py @@ -0,0 +1,34 @@ +"""Статические графики только из сохранённых фактических агрегатов.""" +import json +from pathlib import Path +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +ROOT = Path(__file__).resolve().parents[1] +data = json.loads((ROOT / 'evidence/pipeline.json').read_text(encoding='utf-8')) +points = ('before_publish', 'after_publish', 'before_effect', 'after_original', 'after_variants', 'after_commit') +fig, ax = plt.subplots(figsize=(10, 4.5), layout='constrained') +for index, point in enumerate(points): + values = [row['seconds'] for row in data if row['scenario'] == point] + ax.scatter([index] * len(values), values, s=45) +ax.set_xticks(range(len(points)), points, rotation=15) +ax.set_ylabel('Full scenario duration, seconds') +ax.set_title('Actual crash scenarios: stop, upload, restart, verified downloads') +ax.grid(axis='y', alpha=.25) +fig.savefig(ROOT / 'evidence/faults.png', dpi=160) +plt.close(fig) +data = json.loads((ROOT / 'evidence/load-pipeline.json').read_text(encoding='utf-8')) +fig, axes = plt.subplots(1, 2, figsize=(10, 4), layout='constrained') +for index, profile in enumerate(sorted({row['profile'] for row in data})): + rows = [row for row in data if row['profile'] == profile] + for axis, metric in zip(axes, ('successful_rps', 'p95')): + axis.scatter([index] * len(rows), [row[metric] for row in rows], label=profile) +for axis, label in zip(axes, ('Completed images / second', 'Upload to ready p95, seconds')): + axis.set_xticks(range(len({row['profile'] for row in data})), sorted({row['profile'] for row in data})) + axis.set_ylabel(label) + axis.grid(axis='y', alpha=.25) +fig.suptitle('Closed-loop pipeline: 2 clients, 20 seconds, 3 repeats') +fig.savefig(ROOT / 'evidence/load.png', dpi=160) +plt.close(fig) +print('Saved evidence/faults.png and evidence/load.png') diff --git a/scripts/reproduce.py b/scripts/reproduce.py new file mode 100644 index 00000000..b89768c5 --- /dev/null +++ b/scripts/reproduce.py @@ -0,0 +1,74 @@ +"""Чистый Git-экспорт, новые секреты и отдельный временный Compose-проект.""" +import io +import json +import os +from pathlib import Path +import socket +import shutil +import subprocess +import sys +import tempfile +import uuid +import zipfile + +ROOT = Path(__file__).resolve().parents[1] + + +def remove_temporary(directory, project): + resolved = Path(directory).resolve() + temporary_root = Path(tempfile.gettempdir()).resolve() + if resolved.parent != temporary_root or not resolved.name.startswith(project + '-'): + raise RuntimeError('Temporary directory failed path validation') + if not project.startswith('lyakhonov-lab5-repro-') or resolved == ROOT.resolve(): + raise RuntimeError('Refusing unexpected cleanup target') + shutil.rmtree(resolved) + + +def main(): + name = 'lyakhonov-lab5-repro-' + uuid.uuid4().hex[:8] + directory = Path(tempfile.mkdtemp(prefix=name + '-')) + archive = subprocess.check_output(['git', 'archive', '--format=zip', 'HEAD'], cwd=ROOT) + with zipfile.ZipFile(io.BytesIO(archive)) as contents: + contents.extractall(directory) + with socket.socket() as listener: + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + env = dict(os.environ, LAB5_PORT=str(port), LAB5_FAULT_INJECTION='0', JOB_TIMEOUT_SECONDS='60', JOB_LEASE_SECONDS='90') + services = ('api', 'migrate', 'worker', 'dispatcher', 'staging-cleaner') + images = [name + ':app', name + ':s3'] + override = {'services': {service: {'image': images[0]} for service in services}} + override['services'].update({service: {'image': images[1]} for service in ('s3', 'bucket-init')}) + (directory / 'repro-override.json').write_text(json.dumps(override), encoding='utf-8') + compose = ['docker', 'compose', '-p', name, '-f', 'compose.yaml', '-f', 'repro-override.json'] + events = [] + def run(args, required=True): + result = subprocess.run(args, cwd=directory, env=env, text=True, encoding='utf-8', capture_output=True) + # Только безопасные последние строки: ни inspect .Config.Env, ни secret files. + events.append(dict(command=args, exit_code=result.returncode, output=(result.stdout + result.stderr)[-5000:])) + if result.returncode and required: + raise RuntimeError('reproduction failed: ' + ' '.join(args)) + try: + run([sys.executable, '-X', 'utf8', 'scripts/bootstrap.py']) + run([sys.executable, '-X', 'utf8', 'scripts/check_contract.py']) + run(['docker', 'build', '-t', images[0], '.']) + run(['docker', 'build', '-t', images[1], '-f', 'Dockerfile.s3', '.']) + run([*compose, 'up', '-d']) + run([sys.executable, '-X', 'utf8', 'scripts/wait_ready.py', '--project', name, '--base-url', f'http://127.0.0.1:{port}']) + run([sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', f'http://127.0.0.1:{port}']) + run([*compose, 'ps']) + finally: + run([*compose, 'down', '--volumes', '--remove-orphans'], required=False) + for tag in images: + run(['docker', 'image', 'rm', tag], required=False) + result = dict(project=name, directory=str(directory), port=port, + commit=subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), events=events) + remove_temporary(directory, name) + result['temporary_directory_removed'] = not directory.exists() + (ROOT / 'evidence/reproduction.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8') + if any(event['exit_code'] for event in events): + raise SystemExit('Reproduction or cleanup failed; see evidence/reproduction.json') + print('PASS: clean export, fresh secrets, isolated project; temporary Docker resources removed') + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..1e76668c --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,2 @@ +@echo off +python "%~dp0actions.py" restart diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..c4a7a9fd --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +root="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +cd "$root" +docker compose restart -t 4 db redis s3 broker api dispatcher worker staging-cleaner lb diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..0a94006e --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,2 @@ +@echo off +python "%~dp0actions.py" stop-one diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..c80f3f62 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +root="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +cd "$root" +: "${TARGET_INSTANCE_ID:?}" +for container in $(docker compose ps -q api); do + hostname="$(docker inspect --format '{{.Config.Hostname}}' "$container")" + case "$TARGET_INSTANCE_ID" in + "$hostname"-????????) + docker compose ps -q api | grep -Fx "$container" >/dev/null + docker stop -t 2 "$container" >/dev/null + exit 0 + ;; + esac +done +exit 1 diff --git a/scripts/summarize.py b/scripts/summarize.py new file mode 100644 index 00000000..a9c9e055 --- /dev/null +++ b/scripts/summarize.py @@ -0,0 +1,43 @@ +"""Таблица отчёта из JSON; без ручного переноса значений.""" +import csv +import json +from pathlib import Path +import re + +ROOT = Path(__file__).resolve().parents[1] +load = json.loads((ROOT / 'evidence/load-pipeline.json').read_text(encoding='utf-8')) +faults = json.loads((ROOT / 'evidence/pipeline.json').read_text(encoding='utf-8')) +text = ['| Профиль | Bytes/image | Завершено за 3 повтора | Успешных image/s (min–max) | p50 ready, s | p95 ready, s | p99 ready, s | Ошибки |', + '| --- | --- | --- | --- | --- | --- | --- | --- |'] +for profile in sorted({r['profile'] for r in load}): + rows = [r for r in load if r['profile'] == profile] + def span(key): + return f"{min(r[key] for r in rows):.3f}–{max(r[key] for r in rows):.3f}" + text.append(f"| {profile} | {rows[0]['bytes_per_image']} | {sum(r['completed'] for r in rows)} | {span('successful_rps')} | {span('p50')} | {span('p95')} | {span('p99')} | {sum(len(r['failures']) for r in rows)} |") +def mib(value): + number, unit = re.match(r'([0-9.]+)([A-Za-z]+)', value.strip()).groups() + return float(number) * {'B':1/1048576,'KiB':1/1024,'MiB':1,'GiB':1024,'kB':1000/1048576,'MB':1000000/1048576,'GB':1000000000/1048576}[unit] +snapshots = [s for r in load for s in r['stats']] +peak_cpu = max(sum(float(c['CPUPerc'].strip('%')) for c in s['containers']) for s in snapshots) +peak_memory = max(sum(mib(c['MemUsage'].split('/')[0]) for c in s['containers']) for s in snapshots) +text.extend(['', f'В собранных snapshots максимальная сумма CPU контейнеров: **{peak_cpu:.1f}%** (100% = один CPU); сумма cgroup memory: **{peak_memory:.1f} MiB**. Это максимум редких снимков, не непрерывно измеренный пик.', '']) +points = ('before_publish','after_publish','before_effect','after_original','after_variants','after_commit') +text.extend(['| Точка crash | Повторов | Полная длительность сценария, s (min–max) |', '| --- | --- | --- |']) +for point in points: + rows = [r for r in faults if r['scenario'] == point] + text.append(f"| {point} | {len(rows)} | {min(r['seconds'] for r in rows):.3f}–{max(r['seconds'] for r in rows):.3f} |") +text.extend(['', 'Во всех сохранённых crash/version/replay-сценариях проверки инвариантов прошли. Увеличение входного изображения снижает throughput и повышает время до ready в измеренном профиле. Повторы показывают разброс, а не статистически надёжную оценку хвостов: выборка и длительность малы.']) +path = ROOT / 'REPORT.md' +report = path.read_text(encoding='utf-8') +begin, end = '', '' +block = begin + '\n\n' + '\n'.join(text) + '\n\n' + end +if begin in report: + report = report[:report.index(begin)] + block + report[report.index(end)+len(end):] +else: + report = report.replace('', block) +path.write_text(report, encoding='utf-8') +with (ROOT / 'evidence/measurements.csv').open('w', newline='', encoding='utf-8') as target: + writer = csv.DictWriter(target, fieldnames=['profile','repeat','completed','successful_rps','p50','p95','p99','error_fraction']) + writer.writeheader() + writer.writerows({key:row[key] for key in writer.fieldnames} for row in load) +print('\n'.join(text)) diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 00000000..a4b5173f --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,44 @@ +"""Явная готовность API, stores и процессов; smoke проверяет обработку задания.""" +import argparse +import json +import subprocess +import time +import urllib.error +import urllib.request + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--project', default='lyakhonov-lab5') + parser.add_argument('--base-url', default='http://127.0.0.1:8085') + args = parser.parse_args() + deadline = time.monotonic()+120 + observed = set() + while time.monotonic() < deadline: + ids = subprocess.check_output(['docker','compose','-p',args.project,'ps','-q'], text=True).split() + if not ids: + time.sleep(1) + continue + rows = json.loads(subprocess.check_output(['docker','inspect',*ids], text=True)) + services = {} + for row in rows: + name = row['Config']['Labels']['com.docker.compose.service'] + services.setdefault(name, []).append(row['State']) + running = all(services.get(name) and all(s['Running'] for s in services[name]) + for name in ('api','db','s3','broker','dispatcher','worker','staging-cleaner','lb')) + healthy = all(services.get(name) and all(s.get('Health',{}).get('Status') == 'healthy' for s in services[name]) + for name in ('db','broker','api')) + try: + with urllib.request.urlopen(args.base_url+'/health/ready', timeout=2) as response: + observed.add(response.headers.get('X-Instance-Id')) + except (urllib.error.URLError, TimeoutError, OSError): + pass + if running and healthy and len(services.get('api', [])) >= 2 and len(observed - {None}) >= 2: + print('PASS: two ready API instances, healthy DB/broker, required background containers running') + return + time.sleep(.5) + raise SystemExit('Readiness deadline exceeded') + + +if __name__ == '__main__': + main() diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..3ec4317d --- /dev/null +++ b/src/app.py @@ -0,0 +1,166 @@ +import uuid +import os +import socket +from contextlib import asynccontextmanager +import psycopg +from psycopg_pool import PoolTimeout, TooManyRequests +from starlette.concurrency import run_in_threadpool + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from pydantic import ValidationError as PydanticValidationError +from starlette.exceptions import HTTPException as StarletteHTTPException + +from src.models import ErrorResponse +from src.routes import chats, health, messages, users, attachments +from botocore.exceptions import BotoCoreError, ClientError +from src.storage import storage + + +def create_app() -> FastAPI: + """Создать и настроить приложение FastAPI.""" + @asynccontextmanager + async def lifespan(app): + await run_in_threadpool(storage.pool.open, wait=True) + try: + yield + finally: + await run_in_threadpool(storage.pool.close) + + instance_id = os.getenv("INSTANCE_ID", socket.gethostname()) + "-" + uuid.uuid4().hex[:8] + app = FastAPI(title="MTUSI Messenger Lab5", version="1.5.0", lifespan=lifespan) + + @app.exception_handler(BotoCoreError) + @app.exception_handler(ClientError) + @app.exception_handler(psycopg.OperationalError) + @app.exception_handler(psycopg.errors.QueryCanceled) + @app.exception_handler(psycopg.errors.LockNotAvailable) + @app.exception_handler(PoolTimeout) + @app.exception_handler(TooManyRequests) + async def dependency_error(request, exc): + return JSONResponse(status_code=503, content=ErrorResponse(error={ + "code": "unavailable", "message": "хранилище временно недоступно", + "request_id": request.state.request_id}).model_dump(), + headers={"X-Request-Id": request.state.request_id}) + + @app.middleware("http") + async def add_request_id(request: Request, call_next): + """Добавляет идентификаторы запроса и процесса, ограничивает очередь.""" + request_id = request.headers.get("X-Request-Id", "") or str(uuid.uuid4()) + request.state.request_id = request_id + + # Ограничение числа обслуживаемых запросов в процессе. + if not storage.acquire_request_slot(): + return JSONResponse( + status_code=503, + content=ErrorResponse( + error={ + "code": "unavailable", + "message": "сервер перегружен, попробуйте позже", + "request_id": request_id, + } + ).model_dump(), + headers={"X-Request-Id": request_id, "X-Instance-Id": instance_id}, + ) + + try: + response = await call_next(request) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = instance_id + return response + finally: + storage.release_request_slot() + + @app.exception_handler(StarletteHTTPException) + async def http_exception_handler(request: Request, exc: StarletteHTTPException): + """Обработчик HTTPException: возвращает ошибку в формате контракта {"error": ...}.""" + request_id = getattr(request.state, "request_id", str(uuid.uuid4())) + if isinstance(exc.detail, dict) and "error" in exc.detail: + return JSONResponse( + status_code=exc.status_code, + content=exc.detail, + headers={"X-Request-Id": request_id, **(exc.headers or {})}, + ) + return JSONResponse( + status_code=exc.status_code, + content=ErrorResponse( + error={ + "code": "internal_error", + "message": str(exc.detail), + "request_id": request_id, + } + ).model_dump(), + headers={"X-Request-Id": request_id, "X-Instance-Id": instance_id}, + ) + + @app.exception_handler(RequestValidationError) + async def request_validation_handler(request: Request, exc: RequestValidationError): + """Перехватываем ошибки валидации запроса (422 -> 400 по контракту).""" + request_id = getattr(request.state, "request_id", str(uuid.uuid4())) + errors = exc.errors() + messages = [] + for err in errors: + loc = ".".join(str(x) for x in err.get("loc", [])) + messages.append(f"{loc}: {err.get('msg', 'invalid')}") + return JSONResponse( + status_code=400, + content=ErrorResponse( + error={ + "code": "invalid_request", + "message": "; ".join(messages), + "request_id": request_id, + } + ).model_dump(), + headers={"X-Request-Id": request_id, "X-Instance-Id": instance_id}, + ) + + @app.exception_handler(PydanticValidationError) + async def validation_exception_handler(request: Request, exc: PydanticValidationError): + """Перехватываем ошибки Pydantic (422 -> 400 по контракту).""" + request_id = getattr(request.state, "request_id", str(uuid.uuid4())) + errors = exc.errors() + messages = [] + for err in errors: + loc = ".".join(str(x) for x in err.get("loc", [])) + messages.append(f"{loc}: {err.get('msg', 'invalid')}") + return JSONResponse( + status_code=400, + content=ErrorResponse( + error={ + "code": "invalid_request", + "message": "; ".join(messages), + "request_id": request_id, + } + ).model_dump(), + headers={"X-Request-Id": request_id, "X-Instance-Id": instance_id}, + ) + + @app.exception_handler(Exception) + async def generic_exception_handler(request: Request, exc: Exception): + """Обработчик непредвиденных исключений: скрываем детали, возвращаем 500.""" + request_id = getattr(request.state, "request_id", str(uuid.uuid4())) + return JSONResponse( + status_code=500, + content=ErrorResponse( + error={ + "code": "internal_error", + "message": "внутренняя ошибка сервера", + "request_id": request_id, + } + ).model_dump(), + headers={"X-Request-Id": request_id, "X-Instance-Id": instance_id}, + ) + + # Подключаем роуты + app.include_router(health.router) + app.include_router(users.router) + app.include_router(chats.router) + app.include_router(messages.router) + app.include_router(attachments.router) + + return app + + +# Экземпляр приложения для uvicorn +app = create_app() diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 00000000..68c10390 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,36 @@ +"""RabbitMQ с подтверждаемой публикацией и ручным ack.""" + +import json +from contextlib import contextmanager + +import pika + +from src.config import setting + +QUEUE = 'attachment.process.v1' + + +@contextmanager +def channel(heartbeat=90): + params = pika.URLParameters(setting('BROKER_URL')) + params.heartbeat = heartbeat + params.socket_timeout = 4 + params.stack_timeout = 6 + params.blocked_connection_timeout = 5 + connection = pika.BlockingConnection(params) + try: + result = connection.channel() + result.queue_declare(queue=QUEUE, durable=True) + yield connection, result + finally: + if connection.is_open: + connection.close() + + +def publish(ch, message): + ch.basic_publish( + exchange='', routing_key=QUEUE, body=json.dumps(message, separators=(',', ':')).encode(), + properties=pika.BasicProperties(content_type='application/json', delivery_mode=2, + message_id=message['job_id']), + mandatory=True, + ) diff --git a/src/cache.py b/src/cache.py new file mode 100644 index 00000000..edeb5357 --- /dev/null +++ b/src/cache.py @@ -0,0 +1,60 @@ +"""Необязательный кэш неизменяемого публичного профиля; ошибки обходятся через БД.""" +import logging +import os +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout +import threading +from redis import Redis, RedisError +from pydantic import ValidationError +from src.config import setting +from src.models import User + + +class ProfileCache: + def __init__(self): + url = setting('CACHE_URL') + self.client = Redis.from_url(url, decode_responses=True, socket_connect_timeout=0.2, + socket_timeout=0.2, max_connections=16) if url else None + self.ttl = int(os.getenv('PROFILE_CACHE_TTL_SECONDS','60')) + self.hits = self.misses = self.errors = 0 + # DNS может ждать дольше socket timeout; один фоновый вызов ограничивает и время, и очередь. + self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='profile-cache') + self._pending = None + self._guard = threading.Lock() + + def _bounded(self, action, *args, **kwargs): + with self._guard: + if self._pending is not None and not self._pending.done(): + raise RedisError('кэш занят') + self._pending = self._executor.submit(action, *args, **kwargs) + future = self._pending + try: + return future.result(timeout=0.25) + except FutureTimeout: + raise RedisError('истекло время ожидания кэша') from None + + def get(self, user_id): + if not self.client: + return None + try: + value = self._bounded(self.client.get, 'profile:v1:'+user_id) + if value: + user=User.model_validate_json(value) + if user.id != user_id: + raise ValueError('неверный ключ кэша') + self.hits += 1 + return user + self.misses += 1 + except (RedisError, ValidationError, ValueError): + self.errors += 1 + logging.getLogger(__name__).warning('Кэш профиля недоступен; чтение из PostgreSQL') + return None + + def put(self, user): + if self.client: + try: + self._bounded(self.client.set, 'profile:v1:'+user.id,user.model_dump_json(),ex=self.ttl) + except RedisError: + self.errors += 1 + + +profile_cache = ProfileCache() diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..30d65e7a --- /dev/null +++ b/src/config.py @@ -0,0 +1,10 @@ +"""Настройки окружения и чтение секретов из файлов.""" +import os +from pathlib import Path + + +def setting(name, default=None): + path = os.getenv(name + '_FILE') + if path: + return Path(path).read_text(encoding='utf-8').strip() + return os.getenv(name, default) diff --git a/src/dispatch.py b/src/dispatch.py new file mode 100644 index 00000000..25a78f34 --- /dev/null +++ b/src/dispatch.py @@ -0,0 +1,55 @@ +"""Отдельный relay: lease публикации, publisher confirm, восстановление jobs.""" + +import logging +import signal +import threading +import time + +import pika +import psycopg +from psycopg_pool import PoolTimeout + +from src import broker, jobs, faults +from src.storage import storage + + +def main(): + logging.basicConfig(level=logging.WARNING) + stopping = threading.Event() + for sig in (signal.SIGINT, signal.SIGTERM): + signal.signal(sig, lambda *_: stopping.set()) + storage.pool.open(wait=True) + next_recovery = 0.0 + try: + while not stopping.is_set(): + reserved = None + try: + with broker.channel(heartbeat=10) as (connection, channel): + channel.confirm_delivery() + while not stopping.is_set(): + if time.monotonic() >= next_recovery: + jobs.recover_expired() + next_recovery = time.monotonic() + 1 + reserved = jobs.reserve_publish() + if reserved: + faults.hit('before_publish', reserved) + broker.publish(channel, jobs.envelope(reserved)) + faults.hit('after_publish', reserved) + jobs.confirm_publish(reserved) + reserved = None + else: + connection.process_data_events(time_limit=0.2) + except (pika.exceptions.AMQPError, OSError, psycopg.Error, PoolTimeout) as exc: + logging.warning('dispatch temporarily unavailable: %s', type(exc).__name__) + if reserved: + try: + jobs.publish_failed(reserved) + except (psycopg.Error, PoolTimeout): + pass # expires through the durable publication deadline + stopping.wait(1) + finally: + storage.pool.close() + + +if __name__ == '__main__': + main() diff --git a/src/faults.py b/src/faults.py new file mode 100644 index 00000000..7d26432d --- /dev/null +++ b/src/faults.py @@ -0,0 +1,29 @@ +"""Одноразовые точки отказа. Выключены без LAB5_FAULT_INJECTION=1.""" +import os +import time + +from src.storage import storage + + +def hit(point, job): + if os.getenv('LAB5_FAULT_INJECTION') != '1': + return + with storage.pool.connection() as conn: + fault = conn.execute( + "UPDATE experiment_faults SET hit_at=clock_timestamp(), process_id=%s " + "WHERE job_id=%s AND point=%s AND hit_at IS NULL RETURNING action", + (os.getpid(), job['id'], point), + ).fetchone() + if not fault: + return + if fault['action'] == 'crash': + os._exit(91) # Реальная смерть процесса без finally/ack. + while True: + with storage.pool.connection() as conn: + released = conn.execute( + 'SELECT released FROM experiment_faults WHERE job_id=%s AND point=%s', + (job['id'], point), + ).fetchone()['released'] + if released: + return + time.sleep(0.1) diff --git a/src/image_pipeline.py b/src/image_pipeline.py new file mode 100644 index 00000000..e85ef94a --- /dev/null +++ b/src/image_pipeline.py @@ -0,0 +1,60 @@ +"""Преобразования Pillow в отдельном процессе с жёстким таймаутом родителя.""" + +import json +import sys +from pathlib import Path + +from PIL import Image, ImageOps +from src.pipeline_versions import ENCODING + +Image.MAX_IMAGE_PIXELS = 40_000_000 + + +def fit(image, limit): + width, height = image.size + longest = max(width, height) + if longest <= limit: + return image + size = (max(1, width * limit // longest), max(1, height * limit // longest)) + return image.resize(size, Image.Resampling.LANCZOS) + + +def render(source, target, version=1): + jpeg_quality, webp_quality = ENCODING[version] + with Image.open(source) as opened: + if opened.width * opened.height > Image.MAX_IMAGE_PIXELS: + raise ValueError('pixel limit') + image = ImageOps.exif_transpose(opened) + if 'A' in image.getbands() or 'transparency' in image.info: + rgba = image.convert('RGBA') + rgb = Image.new('RGB', rgba.size, 'white') + rgb.paste(rgba, mask=rgba.getchannel('A')) + else: + rgb = image.convert('RGB') + rgb.info.clear() + del image + thumbnail = fit(rgb, 256) + thumbnail.save(target / 'thumbnail', 'JPEG', quality=jpeg_quality) + crop = ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS) + crop.save(target / 'crop', 'JPEG', quality=jpeg_quality) + optimized = fit(rgb, 1280) + optimized.save(target / 'optimized', 'WEBP', quality=webp_quality, method=4) + return [ + ['thumbnail', 'image/jpeg', *thumbnail.size], + ['crop', 'image/jpeg', *crop.size], + ['optimized', 'image/webp', *optimized.size], + ] + + +def main(): + directory = Path(sys.argv[1]) + try: + result = render(directory / 'original', directory, int(sys.argv[2])) + except (OSError, ValueError, SyntaxError, Image.DecompressionBombError): + return 2 + (directory / 'manifest.json').write_text(json.dumps(result), encoding='utf-8') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 00000000..e5c9ecc5 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,238 @@ +"""Устойчивое намерение обработки, lease и атомарная публикация manifest.""" + +import random +import re +from datetime import datetime +from uuid import UUID, uuid4 + +from src.config import setting +from src.objects import BUCKET, client, object_lock +from src.pipeline_versions import ENCODING +from src.storage import storage, timestamp + +JOB_TIMEOUT = int(setting('JOB_TIMEOUT_SECONDS', '60')) +LEASE_SECONDS = max(JOB_TIMEOUT + 5, int(setting('JOB_LEASE_SECONDS', str(JOB_TIMEOUT + 30)))) + + +def envelope(job): + return { + 'schema_version': 1, + 'job_id': str(job['id']), + 'type': 'attachment.process.v1', + 'attachment_id': str(job['attachment_id']), + 'pipeline_version': job['pipeline_version'], + 'attempt': job['attempt'], + 'created_at': timestamp(job['created_at']), + } + + +def validate(message): + required = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'} + if not isinstance(message, dict) or not required <= message.keys() <= required | {'traceparent', 'tracestate'}: + raise ValueError('invalid fields') + if (type(message['schema_version']) is not int or message['schema_version'] != 1 + or message['type'] != 'attachment.process.v1' + or type(message['pipeline_version']) is not int or message['pipeline_version'] not in ENCODING + or type(message['attempt']) is not int or message['attempt'] < 1): + raise ValueError('invalid job') + UUID(message['job_id']) + UUID(message['attachment_id']) + if datetime.fromisoformat(message['created_at'].replace('Z', '+00:00')).tzinfo is None: + raise ValueError('missing timezone') + if 'traceparent' in message and not re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', + message['traceparent']): + raise ValueError('invalid traceparent') + if 'tracestate' in message and (not isinstance(message['tracestate'], str) or len(message['tracestate']) > 512): + raise ValueError('invalid tracestate') + return {key: message[key] for key in required} + + +def reserve_publish(): + with storage.pool.connection() as conn: + return conn.execute( + """WITH candidate AS ( + SELECT id FROM attachment_jobs WHERE state='queued' + AND next_publish_at <= clock_timestamp() + ORDER BY next_publish_at, id LIMIT 1 FOR UPDATE SKIP LOCKED + ) + UPDATE attachment_jobs j + SET next_publish_at=clock_timestamp()+interval '15 seconds' + FROM candidate WHERE j.id=candidate.id RETURNING j.*""" + ).fetchone() + + +def confirm_publish(job): + with storage.pool.connection() as conn: + conn.execute( + """UPDATE attachment_jobs SET next_publish_at=NULL, published_at=clock_timestamp(), + publish_failures=0 WHERE id=%s AND attempt=%s""", + (job['id'], job['attempt']), + ) + + +def publish_failed(job): + with storage.pool.connection() as conn: + conn.execute( + """UPDATE attachment_jobs SET publish_failures=publish_failures+1, + next_publish_at=clock_timestamp()+%s * interval '1 second' + WHERE id=%s AND attempt=%s AND state='queued' AND next_publish_at IS NOT NULL""", + (delay(job['publish_failures'] + 1), job['id'], job['attempt']), + ) + + +def locked_job(conn, job_id, attachment_id): + attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', + (attachment_id,)).fetchone() + if not attachment: + return None, None + job = conn.execute('SELECT * FROM attachment_jobs WHERE id=%s AND attachment_id=%s FOR UPDATE', + (job_id, attachment_id)).fetchone() + return job, attachment + + +def claim(message): + with storage.pool.connection() as conn: + job, attachment = locked_job(conn, message['job_id'], message['attachment_id']) + if (not job or envelope(job) != message or job['state'] != 'queued' + or job['pipeline_version'] != attachment['target_pipeline_version']): + return None + token = uuid4() + conn.execute( + """UPDATE attachment_jobs SET state='processing', lease_token=%s, + lease_until=clock_timestamp()+%s * interval '1 second', next_publish_at=NULL + WHERE id=%s""", (token, LEASE_SECONDS, job['id']), + ) + conn.execute("UPDATE attachments SET state='processing',updated_at=clock_timestamp() " + "WHERE id=%s AND published_pipeline_version IS NULL", + (job['attachment_id'],)) + return job | {'lease_token': token, 'attachment': attachment} + + +def owns(conn, job, claimed): + return ( + job and (job['state'], job['lease_token']) == ('processing', claimed['lease_token']) + and conn.execute('SELECT clock_timestamp() < %s AS active', (job['lease_until'],)).fetchone()['active'] + ) + + +def complete(claimed, variants): + with storage.pool.connection() as conn: + job, attachment = locked_job(conn, claimed['id'], claimed['attachment_id']) + if not owns(conn, job, claimed) or job['pipeline_version'] != attachment['target_pipeline_version']: + return False + for variant in variants: + conn.execute( + """INSERT INTO attachment_variants + (attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)""", + (job['attachment_id'], job['pipeline_version'], *variant), + ) + conn.execute("UPDATE attachments SET state='ready',published_pipeline_version=%s," + "updated_at=clock_timestamp(),error_code=NULL WHERE id=%s", + (job['pipeline_version'], job['attachment_id'])) + conn.execute("UPDATE attachment_jobs SET state='ready',lease_token=NULL,lease_until=NULL," + "last_error=NULL,settled_at=clock_timestamp() WHERE id=%s", (job['id'],)) + return True + + +def delay(attempt): + return min(30, 2 ** min(attempt - 1, 5)) * random.uniform(0.75, 1.25) + + +def settle_error(conn, job, code, permanent=False): + exhausted = job['attempt'] >= job['attempt_limit'] + if permanent or exhausted: + error = code if permanent else 'retry_exhausted' + conn.execute("UPDATE attachments SET state='failed',error_code=%s,updated_at=clock_timestamp() " + "WHERE id=%s AND published_pipeline_version IS NULL", (error, job['attachment_id'])) + conn.execute("UPDATE attachment_jobs SET state='failed',lease_token=NULL,lease_until=NULL," + "last_error=%s,next_publish_at=NULL,settled_at=clock_timestamp() WHERE id=%s", + (code, job['id'])) + return + conn.execute("UPDATE attachments SET state='queued',updated_at=clock_timestamp() " + "WHERE id=%s AND published_pipeline_version IS NULL", + (job['attachment_id'],)) + conn.execute("""UPDATE attachment_jobs SET state='queued',lease_token=NULL,lease_until=NULL, + last_error=%s,attempt=attempt+1,published_at=NULL, + next_publish_at=clock_timestamp()+%s*interval '1 second' WHERE id=%s""", + (code, delay(job['attempt']), job['id'])) + + +def fail(claimed, code, permanent=False): + with storage.pool.connection() as conn: + job, _ = locked_job(conn, claimed['id'], claimed['attachment_id']) + if owns(conn, job, claimed): + settle_error(conn, job, code, permanent) + + +def recover_expired(): + with storage.pool.connection() as conn: + expired = conn.execute("SELECT id,attachment_id FROM attachment_jobs " + "WHERE lease_until <= clock_timestamp() ORDER BY lease_until LIMIT 16").fetchall() + for candidate in expired: + with storage.pool.connection() as conn: + job, _ = locked_job(conn, candidate['id'], candidate['attachment_id']) + if (job and job['state'] == 'processing' + and not owns(conn, job, job)): + settle_error(conn, job, 'lease_expired') + + +def replay(attachment_id): + with storage.pool.connection() as conn: + attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', + (attachment_id,)).fetchone() + if not attachment: + return None + job = conn.execute('SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE', + (attachment_id, attachment['target_pipeline_version'])).fetchone() + if not job or job['state'] != 'failed': + return None + source = attachment['object_key'] if attachment['published_pipeline_version'] else job['staging_key'] + object_lock(conn, source) + client().head_object(Bucket=BUCKET, Key=source) + new_attempt = job['attempt'] + 1 + budget = int(setting('JOB_MAX_ATTEMPTS', '5')) + conn.execute( + """UPDATE attachment_jobs SET state='queued', attempt=%s, attempt_limit=%s, + next_publish_at=clock_timestamp(), published_at=NULL, publish_failures=0, settled_at=NULL + WHERE id=%s""", (new_attempt, new_attempt + budget - 1, job['id']), + ) + conn.execute("INSERT INTO attachment_replays(job_id,attempt,reason) VALUES (%s,%s,'operator')", + (job['id'], new_attempt)) + conn.execute("UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s AND published_pipeline_version IS NULL", (attachment_id,)) + return job['id'], new_attempt + + +def upgrade(attachment_id, version): + with storage.pool.connection() as conn: + attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,)).fetchone() + if not attachment or version not in ENCODING or version <= attachment['target_pipeline_version']: + return None + previous = conn.execute( + 'SELECT staging_key FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s', + (attachment_id, attachment['target_pipeline_version']), + ).fetchone() + if not previous and not attachment['published_pipeline_version']: + return None + source = attachment['object_key'] if attachment['published_pipeline_version'] else previous['staging_key'] + object_lock(conn, source) + client().head_object(Bucket=BUCKET, Key=source) + conn.execute( + """UPDATE attachment_jobs SET state='superseded', lease_token=NULL, lease_until=NULL, + next_publish_at=NULL, settled_at=clock_timestamp() + WHERE attachment_id=%s AND state IN ('queued','processing')""", (attachment_id,), + ) + conn.execute( + """UPDATE attachments SET target_pipeline_version=%s, + state=CASE WHEN published_pipeline_version IS NULL THEN 'queued' ELSE 'ready' END, + error_code=NULL,updated_at=clock_timestamp() WHERE id=%s""", + (version, attachment_id), + ) + job_id = uuid4() + conn.execute( + """INSERT INTO attachment_jobs(id,attachment_id,staging_key,attempt_limit,next_publish_at,pipeline_version) + VALUES (%s,%s,%s,%s,clock_timestamp(),%s)""", + (job_id, attachment_id, f'staging/{attachment_id}', int(setting('JOB_MAX_ATTEMPTS', '5')), version), + ) + return job_id diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..69dacbfe --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,19 @@ +"""Версионируемые миграции: одна транзакция, блокировка от параллельного запуска.""" +from pathlib import Path +import psycopg +from src.config import setting + + +def main(): + with psycopg.connect(setting('DATABASE_URL'), connect_timeout=5) as conn: + conn.execute('SELECT pg_advisory_xact_lock(2301002)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY)') + for path in sorted(Path('/app/migrations' if Path('/app/migrations').exists() else 'migrations').glob('*.sql')): + if not conn.execute('SELECT 1 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone(): + conn.execute(path.read_text(encoding='utf-8')) + conn.execute('INSERT INTO schema_migrations VALUES (%s)', (path.name,)) + print('Применена миграция:', path.name) + + +if __name__ == '__main__': + main() diff --git a/src/models.py b/src/models.py new file mode 100644 index 00000000..dc507cce --- /dev/null +++ b/src/models.py @@ -0,0 +1,126 @@ +from datetime import datetime, timezone +from typing import Optional +import uuid + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +def utc_now() -> str: + """Возвращает текущее время в формате RFC 3339 UTC с суффиксом Z.""" + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +class ErrorDetail(BaseModel): + """Детали ошибки API.""" + code: str + message: str + request_id: str + + +class ErrorResponse(BaseModel): + """Стандартный ответ об ошибке.""" + error: ErrorDetail + + +class Health(BaseModel): + """Ответ проверки работоспособности.""" + status: str = "ok" + + +class User(BaseModel): + """Модель пользователя.""" + id: str + username: str + display_name: str + created_at: str + + +class CreateUser(BaseModel): + """Данные для создания пользователя.""" + model_config = ConfigDict(extra="forbid", strict=True) + + password: str = Field(..., min_length=12, max_length=128) + username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(..., min_length=1, max_length=100) + + @field_validator("display_name") + @classmethod + def display_name_not_whitespace_only(cls, v: str) -> str: + """Запрещаем display_name, состоящий только из пробельных символов.""" + if v.strip() == "": + raise ValueError("display_name не может состоять только из пробелов") + return v + + +class Chat(BaseModel): + """Модель чата.""" + id: str + title: str + member_ids: list[str] + created_at: str + + +class CreateChat(BaseModel): + """Данные для создания чата.""" + model_config = ConfigDict(extra="forbid", strict=True) + + title: str = Field(..., min_length=1, max_length=100) + member_ids: list[str] = Field(..., min_length=1, max_length=100) + + @field_validator("member_ids") + @classmethod + def valid_members(cls, values): + """Проверить UUID и привести их к единому представлению.""" + return [str(uuid.UUID(value)) for value in values] + + @field_validator("title") + @classmethod + def title_not_whitespace_only(cls, v: str) -> str: + """Запрещаем title, состоящий только из пробельных символов.""" + if v.strip() == "": + raise ValueError("title не может состоять только из пробелов") + return v + + +class Message(BaseModel): + """Модель сообщения.""" + id: str + chat_id: str + sender_id: str + text: str + created_at: str + attachment_ids: list[str] = Field(default_factory=list) + + +class CreateMessage(BaseModel): + """Данные для отправки сообщения.""" + model_config = ConfigDict(extra="forbid", strict=True) + + text: str = Field(..., max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator('attachment_ids') + @classmethod + def valid_attachments(cls, values): + ids = [str(uuid.UUID(v)) for v in values] + if len(set(ids)) != len(ids): + raise ValueError('duplicate attachment_id') + return ids + + @model_validator(mode='after') + def valid_content(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError('empty message') + return self + + +class ChatPage(BaseModel): + """Страница списка чатов с курсорной пагинацией.""" + items: list[Chat] + next_cursor: Optional[str] = None + + +class MessagePage(BaseModel): + """Страница истории сообщений с курсорной пагинацией.""" + items: list[Message] + next_cursor: Optional[str] = None diff --git a/src/objects.py b/src/objects.py new file mode 100644 index 00000000..d3ec1dab --- /dev/null +++ b/src/objects.py @@ -0,0 +1,36 @@ +"""S3 operations with bounded timeouts and immutable server-generated keys.""" +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError +from src.config import setting + +def client(): + return boto3.client('s3', endpoint_url=setting('S3_ENDPOINT','http://s3:9000'), + aws_access_key_id=setting('S3_ACCESS_KEY'), aws_secret_access_key=setting('S3_SECRET_KEY'), + region_name='us-east-1', config=Config(connect_timeout=2,read_timeout=5, + retries={'total_max_attempts':1},s3={'addressing_style':'path'})) + +BUCKET = setting('S3_BUCKET','attachments') + + +class ObjectConflict(Exception): + """Неизменяемый ключ уже содержит другие данные.""" + + +def put_immutable(s3, key, stream, size, content_type, digest): + try: + s3.put_object(Bucket=BUCKET, Key=key, Body=stream, ContentLength=size, + ContentType=content_type, Metadata={'sha256': digest}, IfNoneMatch='*') + except ClientError as exc: + if exc.response['ResponseMetadata']['HTTPStatusCode'] != 412: + raise + existing = s3.head_object(Bucket=BUCKET, Key=key) + if (existing['ContentLength'], existing['ContentType'], existing['Metadata'].get('sha256')) != ( + size, content_type, digest): + raise ObjectConflict(key) from None + + +def object_lock(conn, key, wait=True): + fn = 'pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock' + row = conn.execute('SELECT '+fn+'(hashtextextended(%s,42)) AS locked',(key,)).fetchone() + return wait or row['locked'] diff --git a/src/operator.py b/src/operator.py new file mode 100644 index 00000000..622f2b82 --- /dev/null +++ b/src/operator.py @@ -0,0 +1,47 @@ +"""Локальные операции над реестром заданий.""" + +import argparse +import json +from uuid import UUID + +from botocore.exceptions import BotoCoreError, ClientError + +from src import jobs +from src.storage import storage, timestamp + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('action', choices=('failed', 'replay', 'upgrade')) + parser.add_argument('attachment_id', type=UUID, nargs='?') + parser.add_argument('--version', type=int, default=2) + args = parser.parse_args() + if args.action in ('replay', 'upgrade') and args.attachment_id is None: + parser.error('operation requires attachment_id') + storage.pool.open(wait=True) + try: + if args.action in ('replay', 'upgrade'): + try: + if args.action == 'replay': + result = jobs.replay(args.attachment_id) + else: + job_id = jobs.upgrade(args.attachment_id, args.version) + result = (job_id, 1) if job_id else None + except (BotoCoreError, ClientError): + raise SystemExit('Источник обработки недоступен') from None + if result is None: + raise SystemExit('Операция неприменима к текущему состоянию или версии') + print(json.dumps({'job_id': str(result[0]), 'attempt': result[1]})) + else: + with storage.pool.connection() as conn: + rows = conn.execute( + """SELECT id, attachment_id, pipeline_version, attempt, last_error, settled_at + FROM attachment_jobs WHERE state='failed' ORDER BY settled_at DESC NULLS LAST LIMIT 100""" + ).fetchall() + print(json.dumps(rows, default=lambda value: str(value) if isinstance(value, UUID) else timestamp(value))) + finally: + storage.pool.close() + + +if __name__ == '__main__': + main() diff --git a/src/pipeline_versions.py b/src/pipeline_versions.py new file mode 100644 index 00000000..89ed8e1c --- /dev/null +++ b/src/pipeline_versions.py @@ -0,0 +1,6 @@ +"""Версии преобразований и операция перехода на следующий manifest.""" + +ENCODING = { + 1: (85, 82), + 2: (90, 88), +} diff --git a/src/routes/__init__.py b/src/routes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/routes/attachments.py b/src/routes/attachments.py new file mode 100644 index 00000000..a2728201 --- /dev/null +++ b/src/routes/attachments.py @@ -0,0 +1,157 @@ +"""Bounded synchronous image uploads; one decode per process.""" +import asyncio +import hashlib +import tempfile +import unicodedata +import uuid +import warnings +from PIL import Image, UnidentifiedImageError +from fastapi import APIRouter, Depends, HTTPException, Request +from starlette.concurrency import run_in_threadpool +from starlette.responses import StreamingResponse +from starlette.requests import ClientDisconnect +from src.routes.common import require_user, make_error, get_request_id +from src.storage import storage, timestamp +from src.objects import client, BUCKET, object_lock, put_immutable +from src.config import setting + +router = APIRouter() +slots = asyncio.Semaphore(1) +MAX_BYTES=10*1024*1024 +Image.MAX_IMAGE_PIXELS=40_000_000 + +class InvalidImage(Exception): + def __init__(self,status): + self.status=status + +def error(request, status, code, message): + return HTTPException(status, detail=make_error(code,message,get_request_id(request)).model_dump()) + +def inspect_image(stream, declared): + try: + with warnings.catch_warnings(): + warnings.simplefilter('error', Image.DecompressionBombWarning) + with Image.open(stream) as image: + if image.width*image.height>40_000_000: + raise InvalidImage(413) + actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(image.format) + if not actual or actual != declared: + raise InvalidImage(415) + image.verify() + stream.seek(0) + with Image.open(stream) as image: + image.load() + except (Image.DecompressionBombError,Image.DecompressionBombWarning): + raise InvalidImage(413) from None + except (UnidentifiedImageError,OSError,SyntaxError,ValueError): + raise InvalidImage(415) from None + finally: + stream.seek(0) + +def metadata(row, variants=()): + return {k: str(row[k]) for k in ('id','chat_id','owner_id')} | { + k:row[k] for k in ('filename','content_type','size_bytes','sha256')} | { + 'status':row['state'],'variants':list(variants),'error_code':row['error_code'], + 'created_at':timestamp(row['created_at']),'updated_at':timestamp(row['updated_at'])} + +def save(stream, chat_id, owner, filename, content_type, size, digest): + attachment_id=uuid.uuid4() + key=f'originals/{attachment_id}' + staging_key=f'staging/{attachment_id}' + with storage.pool.connection() as conn: + object_lock(conn, staging_key) + put_immutable(client(), staging_key, stream, size, content_type, digest) + row=conn.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *", + (attachment_id,chat_id,owner,key,filename,content_type,size,digest)).fetchone() + conn.execute( + "INSERT INTO attachment_jobs(id,attachment_id,staging_key,attempt_limit,next_publish_at) " + "VALUES (%s,%s,%s,%s,clock_timestamp())", + (uuid.uuid4(), attachment_id, staging_key, int(setting('JOB_MAX_ATTEMPTS', '5'))), + ) + return metadata(row) + +@router.post('/api/v1/chats/{chat_id}/attachments',status_code=202) +async def upload(request:Request,chat_id:uuid.UUID,filename:str='',actor:str=Depends(require_user)): + if not await run_in_threadpool(storage.is_chat_member,str(chat_id),actor): + raise error(request,404,'not_found','chat not found') + if not 1<=len(filename)<=255 or any(c in '/\\' or unicodedata.category(c)=='Cc' for c in filename): + raise error(request,400,'invalid_request','invalid filename') + mime=request.headers.get('content-type','').lower() + if mime not in ('image/png','image/jpeg'): + raise error(request,415,'unsupported_media_type','PNG/JPEG required') + length=request.headers.get('content-length') + if length is not None: + if not length.isdecimal(): + raise error(request,400,'invalid_request','invalid length') + if int(length)>MAX_BYTES: + raise error(request,413,'payload_too_large','limit 10 MiB') + try: + await asyncio.wait_for(slots.acquire(),timeout=25) + except asyncio.TimeoutError: + raise error(request,503,'unavailable','upload queue busy') from None + try: + with tempfile.TemporaryFile() as stream: + size=0; digest=hashlib.sha256() + async def receive(): + nonlocal size + async for chunk in request.stream(): + size+=len(chunk) + if size>MAX_BYTES: + raise error(request,413,'payload_too_large','limit 10 MiB') + digest.update(chunk) + await run_in_threadpool(stream.write,chunk) + try: + await asyncio.wait_for(receive(),timeout=20) + await run_in_threadpool(stream.seek,0) + await run_in_threadpool(inspect_image,stream,mime) + except ClientDisconnect: + raise error(request,400,'invalid_request','upload disconnected') from None + except asyncio.TimeoutError: + raise error(request,400,'invalid_request','upload timed out') from None + except InvalidImage as exc: + raise error(request,exc.status,'payload_too_large' if exc.status==413 else 'unsupported_media_type','invalid image') from None + return await run_in_threadpool(save,stream,str(chat_id),actor,filename,mime,size,digest.hexdigest()) + finally: + slots.release() + +def accessible(conn, request, attachment_id, actor): + # One consistent manifest for all reads in the enclosing transaction. + row=conn.execute('SELECT a.* FROM attachments a JOIN chat_members m ON a.chat_id=m.chat_id ' + 'WHERE a.id=%s AND m.user_id=%s FOR SHARE OF a',(attachment_id,actor)).fetchone() + if not row: + raise error(request,404,'not_found','attachment not found') + return row + +@router.get('/api/v1/attachments/{attachment_id}') +def get_attachment(request:Request,attachment_id:uuid.UUID,actor:str=Depends(require_user)): + with storage.pool.connection() as conn: + row = accessible(conn, request, attachment_id, actor) + variants = conn.execute( + 'SELECT name,content_type,size_bytes,sha256,width,height FROM attachment_variants ' + 'WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name', + (attachment_id, row['published_pipeline_version']), + ).fetchall() + return metadata(row, variants) + +@router.get('/api/v1/attachments/{attachment_id}/content') +def content(request:Request,attachment_id:uuid.UUID,variant:str='original',actor:str=Depends(require_user)): + with storage.pool.connection() as conn: + row = accessible(conn, request, attachment_id, actor) + if variant not in ('original','thumbnail','crop','optimized'): + raise error(request,400,'invalid_request','invalid variant') + if row['state'] != 'ready': + raise error(request,409,'conflict','attachment not ready') + if variant != 'original': + row = conn.execute( + 'SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s', + (attachment_id, row['published_pipeline_version'], variant), + ).fetchone() + if not row: + raise error(request,404,'not_found','variant not found') + obj=client().get_object(Bucket=BUCKET,Key=row['object_key']) + def chunks(): + try: + yield from obj['Body'].iter_chunks(chunk_size=64*1024) + finally: + obj['Body'].close() + return StreamingResponse(chunks(),media_type=row['content_type'],headers={'Content-Length':str(row['size_bytes'])}) diff --git a/src/routes/chats.py b/src/routes/chats.py new file mode 100644 index 00000000..1324eca1 --- /dev/null +++ b/src/routes/chats.py @@ -0,0 +1,79 @@ +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Request + +from src.models import Chat, ChatPage, CreateChat +from src.routes.common import get_request_id, make_error, require_user +from src.storage import LimitExceeded, ValidationError, storage + +router = APIRouter() + + +@router.post("/api/v1/chats", response_model=Chat, status_code=201) +def create_chat( + request: Request, + body: CreateChat, + x_user_id: str = Depends(require_user), +): + """Создать чат. Создатель обязательно должен входить в member_ids.""" + request_id = get_request_id(request) + + try: + chat = storage.create_chat(body.title, body.member_ids, x_user_id) + return chat + except ValidationError as e: + raise HTTPException( + status_code=400, + detail=make_error("invalid_request", str(e), request_id).model_dump(), + ) + except LimitExceeded as e: + raise HTTPException( + status_code=429, + detail=make_error("rate_limited", str(e), request_id).model_dump(), + ) + + +@router.get("/api/v1/chats", response_model=ChatPage) +def list_chats( + request: Request, + x_user_id: str = Depends(require_user), + limit: int = 50, + cursor: str | None = None, +): + """Список чатов текущего пользователя с курсорной пагинацией.""" + request_id = get_request_id(request) + + if limit < 1 or limit > 100: + raise HTTPException( + status_code=400, + detail=make_error("invalid_request", "limit должен быть от 1 до 100", request_id).model_dump(), + ) + + try: + items, next_cursor = storage.list_chats_for_user(x_user_id, cursor, limit) + return ChatPage(items=items, next_cursor=next_cursor) + except ValidationError as e: + raise HTTPException( + status_code=400, + detail=make_error("invalid_request", str(e), request_id).model_dump(), + ) + + +@router.get("/api/v1/chats/{chat_id}", response_model=Chat) +def get_chat( + request: Request, + chat_id: UUID, + x_user_id: str = Depends(require_user), +): + """Получить чат по id. Доступен только участникам чата (чужой — 404).""" + request_id = get_request_id(request) + + chat_id_str = str(chat_id) + chat = storage.get_chat(chat_id_str) + if not chat or x_user_id not in chat.member_ids: + raise HTTPException( + status_code=404, + detail=make_error("not_found", "чат не найден", request_id).model_dump(), + ) + + return chat diff --git a/src/routes/common.py b/src/routes/common.py new file mode 100644 index 00000000..628e51e6 --- /dev/null +++ b/src/routes/common.py @@ -0,0 +1,29 @@ +"""Общие ошибки и проверка Bearer-сессии; X-User-Id не используется.""" +from fastapi import HTTPException, Request +from src.models import ErrorResponse +from src.storage import storage + + +def make_error(code, message, request_id): + return ErrorResponse(error={'code': code, 'message': message, 'request_id': request_id}) + + +def get_request_id(request): + return request.state.request_id + + +def unauthorized(request, basic=False): + challenge = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' + raise HTTPException(401, detail=make_error('unauthorized', 'неверные учётные данные', get_request_id(request)).model_dump(), headers={'WWW-Authenticate': challenge}) + + +def require_user(request: Request): + authorization = request.headers.get('Authorization', '') + parts = authorization.split() + if len(parts) != 2 or parts[0].lower() != 'bearer': + unauthorized(request) + user_id = storage.session_user(parts[1]) + if not user_id: + unauthorized(request) + request.state.token = parts[1] + return user_id diff --git a/src/routes/health.py b/src/routes/health.py new file mode 100644 index 00000000..9e45f954 --- /dev/null +++ b/src/routes/health.py @@ -0,0 +1,23 @@ +"""Жизнь процесса и готовность критичного хранилища проверяются отдельно.""" +from fastapi import APIRouter +from src.models import Health +from src.storage import storage +router = APIRouter() + + +@router.get('/health/live') +def liveness(): + return Health() + + +@router.get('/health/ready') +def readiness(): + storage.ready() + from src.objects import client, BUCKET + client().list_objects_v2(Bucket=BUCKET, Prefix="staging/", MaxKeys=1) + return Health() + + +@router.get('/stats') +def stats(): + return storage.get_stats() diff --git a/src/routes/messages.py b/src/routes/messages.py new file mode 100644 index 00000000..1045e5dc --- /dev/null +++ b/src/routes/messages.py @@ -0,0 +1,85 @@ +from uuid import UUID +import re +import os + +from fastapi import APIRouter, Depends, HTTPException, Request + +from src.models import CreateMessage, Message, MessagePage +from src.routes.common import get_request_id, make_error, require_user +from src.storage import NotFoundError, ConflictError, LimitExceeded, ValidationError, storage + +router = APIRouter() + + +@router.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=201) +def create_message( + request: Request, + chat_id: UUID, + body: CreateMessage, + x_user_id: str = Depends(require_user), +): + """Отправить сообщение от владельца Bearer-сессии с общей защитой повторов.""" + request_id = get_request_id(request) + + chat_id_str = str(chat_id) + if os.getenv('OPTIMIZED_WRITES','1') == '1': + allowed = storage.is_chat_member(chat_id_str, x_user_id) + else: + chat = storage.get_chat(chat_id_str) + allowed = bool(chat and x_user_id in chat.member_ids) + if not allowed: + raise HTTPException( + status_code=404, + detail=make_error("not_found", "чат не найден", request_id).model_dump(), + ) + + try: + key = request.headers.get('Idempotency-Key', '') + if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + raise HTTPException(400, detail=make_error('invalid_request', 'неверный Idempotency-Key', request_id).model_dump()) + msg = storage.create_message(chat_id_str, x_user_id, body.text, key, body.attachment_ids) + return msg + except NotFoundError as e: + raise HTTPException(404, detail=make_error('not_found', str(e), request_id).model_dump()) + except ConflictError as e: + raise HTTPException(409, detail=make_error('conflict', str(e), request_id).model_dump()) + except LimitExceeded as e: + raise HTTPException( + status_code=429, + detail=make_error("rate_limited", str(e), request_id).model_dump(), + ) + + +@router.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage) +def list_messages( + request: Request, + chat_id: UUID, + x_user_id: str = Depends(require_user), + limit: int = 50, + cursor: str | None = None, +): + """История сообщений чата с курсорной пагинацией. Доступно только участникам.""" + request_id = get_request_id(request) + + chat_id_str = str(chat_id) + chat = storage.get_chat(chat_id_str) + if not chat or x_user_id not in chat.member_ids: + raise HTTPException( + status_code=404, + detail=make_error("not_found", "чат не найден", request_id).model_dump(), + ) + + if limit < 1 or limit > 100: + raise HTTPException( + status_code=400, + detail=make_error("invalid_request", "limit должен быть от 1 до 100", request_id).model_dump(), + ) + + try: + items, next_cursor = storage.list_messages(chat_id_str, cursor, limit) + return MessagePage(items=items, next_cursor=next_cursor) + except ValidationError as e: + raise HTTPException( + status_code=400, + detail=make_error("invalid_request", str(e), request_id).model_dump(), + ) diff --git a/src/routes/users.py b/src/routes/users.py new file mode 100644 index 00000000..4709e263 --- /dev/null +++ b/src/routes/users.py @@ -0,0 +1,49 @@ +"""Регистрация, вход, текущий пользователь и отзыв сессии.""" +import base64 +import binascii +from fastapi import APIRouter, Request, Response, HTTPException +from src.models import CreateUser, User +from src.routes.common import get_request_id, make_error, require_user, unauthorized +from src.storage import ConflictError, storage + +router = APIRouter() + + +@router.post('/api/v1/users', response_model=User, status_code=201) +def register_user(request: Request, body: CreateUser): + try: + return storage.create_user(body.username,body.display_name,body.password) + except ConflictError as exc: + raise HTTPException(409,detail=make_error('conflict',str(exc),get_request_id(request)).model_dump()) + + +@router.post('/api/v1/auth/sessions', status_code=201) +async def login(request: Request): + if await request.body(): + raise HTTPException(400, detail=make_error('invalid_request','тело запроса не допускается',get_request_id(request)).model_dump()) + parts = request.headers.get('Authorization','').split() + try: + if len(parts) != 2 or parts[0].lower() != 'basic': + unauthorized(request, basic=True) + username,password = base64.b64decode(parts[1],validate=True).decode('utf-8').split(':',1) + username.encode('ascii') + except (ValueError, UnicodeError, binascii.Error): + unauthorized(request,basic=True) + # Вычисление KDF и синхронные запросы выполняются вне цикла событий. + from starlette.concurrency import run_in_threadpool + result = await run_in_threadpool(storage.login,username,password) + if not result: + unauthorized(request,basic=True) + return result + + +@router.get('/api/v1/users/me',response_model=User) +def current_user(request: Request): + return storage.get_user(require_user(request)) + + +@router.delete('/api/v1/auth/sessions/current',status_code=204) +def logout(request: Request): + require_user(request) + storage.logout(request.state.token) + return Response(status_code=204) diff --git a/src/staging.py b/src/staging.py new file mode 100644 index 00000000..26ec0ad9 --- /dev/null +++ b/src/staging.py @@ -0,0 +1,59 @@ +"""Очистка staging с блокировкой, общей с HTTP upload и replay.""" + +import logging +import signal +import threading +from datetime import datetime, timezone + +import psycopg +from botocore.exceptions import BotoCoreError, ClientError +from psycopg_pool import PoolTimeout + +from src.config import setting +from src.objects import BUCKET, client, object_lock +from src.storage import storage + + +def cleanup(s3, retention): + count = 0 + now = datetime.now(timezone.utc) + pages = s3.get_paginator('list_objects_v2') + for page in pages.paginate(Bucket=BUCKET, Prefix='staging/'): + for item in page.get('Contents', []): + key = item['Key'] + with storage.pool.connection() as conn: + if not object_lock(conn, key, wait=False): + continue + job = conn.execute( + 'SELECT state,settled_at FROM attachment_jobs WHERE staging_key=%s ' + 'ORDER BY pipeline_version DESC LIMIT 1', (key,), + ).fetchone() + retention_start = (job['settled_at'] if job else None) or item['LastModified'] + age = (now - retention_start).total_seconds() + if job and job['state'] in ('queued', 'processing'): + continue + if (job and job['state'] == 'ready') or age >= retention: + s3.delete_object(Bucket=BUCKET, Key=key) + count += 1 + return count + + +def main(): + stop = threading.Event() + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: stop.set()) + retention = max(60, int(setting('STAGING_RETENTION_SECONDS', '86400'))) + storage.pool.open(wait=True) + try: + while not stop.is_set(): + try: + cleanup(client(), retention) + except (BotoCoreError, ClientError, OSError, psycopg.Error, PoolTimeout) as exc: + logging.warning('staging cleanup temporarily unavailable: %s', type(exc).__name__) + stop.wait(30) + finally: + storage.pool.close() + + +if __name__ == '__main__': + main() diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..45ad1a94 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,276 @@ +"""PostgreSQL — источник истины; каждый контекст подключения завершает транзакцию.""" +import base64 +import binascii +from datetime import datetime, timezone +import hashlib +import json +import os +import secrets +import threading +import uuid + +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError, VerificationError, InvalidHashError +from psycopg.errors import UniqueViolation, ForeignKeyViolation +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from src.config import setting +from src.cache import profile_cache +from src.models import User, Chat, Message + +_CURSOR_FIELDS = {"scope", "resource_id", "created_at", "id"} +PASSWORDS = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +DUMMY_HASH = PASSWORDS.hash('synthetic-dummy-password') + + +class ValidationError(Exception): + """Неверный запрос.""" + + +class ConflictError(Exception): + """Имя уже занято.""" + + +class LimitExceeded(Exception): + """Перегрузка сервера.""" + + +class NotFoundError(Exception): + """Объект отсутствует или недоступен.""" + + +def timestamp(value): + return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def public_row(row): + return {k: timestamp(v) if isinstance(v, datetime) else str(v) if isinstance(v, uuid.UUID) else v + for k, v in row.items() if k != 'password_hash'} + + +class PostgresStorage: + def __init__(self): + self.pool = ConnectionPool(setting('DATABASE_URL', ''), min_size=1, max_size=int(os.getenv("POOL_SIZE", "10")), + timeout=3, max_waiting=32, open=False, check=ConnectionPool.check_connection, + kwargs={'row_factory': dict_row, 'connect_timeout': 3, + 'options': '-c statement_timeout=3000 -c lock_timeout=2000'}) + self._lock = threading.Lock() + self._requests = 0 + + @staticmethod + def _encode_cursor(scope: str, resource_id: str, created_at: str, item_id: str) -> str: + """Собрать непрозрачный курсор, привязанный к конкретному ресурсу.""" + payload = { + "scope": scope, + "resource_id": resource_id, + "created_at": created_at, + "id": item_id, + } + raw = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii") + + @staticmethod + def _decode_cursor(cursor: str, expected_scope: str, expected_resource_id: str) -> tuple[str, str]: + """Разобрать и проверить курсор. + + Клиент видит курсор как непрозрачную строку, поэтому любые пустые, + повреждённые или относящиеся к другому ресурсу значения отклоняем. + """ + if cursor == "": + raise ValidationError("некорректный cursor") + + allowed_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_=" + if any(char not in allowed_chars for char in cursor) or "=" in cursor.rstrip("="): + raise ValidationError("некорректный cursor") + + try: + raw = base64.b64decode(cursor.encode("ascii"), altchars=b"-_", validate=True) + payload = json.loads(raw.decode("utf-8")) + except (UnicodeEncodeError, UnicodeDecodeError, binascii.Error, json.JSONDecodeError): + raise ValidationError("некорректный cursor") + + if not isinstance(payload, dict) or set(payload) != _CURSOR_FIELDS: + raise ValidationError("некорректный cursor") + + if not all(isinstance(payload[field], str) for field in _CURSOR_FIELDS): + raise ValidationError("некорректный cursor") + + if payload["scope"] != expected_scope or payload["resource_id"] != expected_resource_id: + raise ValidationError("некорректный cursor") + + try: + uuid.UUID(payload["resource_id"]) + uuid.UUID(payload["id"]) + datetime.fromisoformat(payload["created_at"].replace("Z", "+00:00")) + except ValueError: + raise ValidationError("некорректный cursor") + + return payload["created_at"], payload["id"] + + + def create_user(self, username, display_name, password): + password_hash = PASSWORDS.hash(password) + try: + with self.pool.connection() as conn: + row = conn.execute('INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at', + (uuid.uuid4(), username, display_name, password_hash)).fetchone() + return User(**public_row(row)) + except UniqueViolation: + raise ConflictError('имя пользователя уже занято') from None + + def get_user(self, user_id): + cached = profile_cache.get(user_id) + if cached: + return cached + with self.pool.connection() as conn: + row = conn.execute('SELECT id,username,display_name,created_at FROM users WHERE id=%s', (user_id,)).fetchone() + user = User(**public_row(row)) if row else None + if user: + profile_cache.put(user) + return user + + def login(self, username, password): + with self.pool.connection() as conn: + row = conn.execute('SELECT * FROM users WHERE username=%s', (username,)).fetchone() + try: + PASSWORDS.verify(row['password_hash'] if row else DUMMY_HASH, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return None + if not row: + return None + token = secrets.token_urlsafe(32) + with self.pool.connection() as conn: + session = conn.execute("INSERT INTO sessions VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') RETURNING expires_at", + (self.token_hash(token), row['id'], int(os.getenv('SESSION_TTL_SECONDS', '3600')))).fetchone() + return {'token': token, 'token_type': 'Bearer', 'expires_at': timestamp(session['expires_at']), 'user': User(**public_row(row)).model_dump()} + + @staticmethod + def token_hash(token): + return hashlib.sha256(token.encode()).hexdigest() + + def session_user(self, token): + with self.pool.connection() as conn: + row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()', (self.token_hash(token),)).fetchone() + return str(row['user_id']) if row else None + + def logout(self, token): + with self.pool.connection() as conn: + conn.execute('DELETE FROM sessions WHERE token_hash=%s', (self.token_hash(token),)) + + def create_chat(self, title, member_ids, creator_id): + if len(set(member_ids)) != len(member_ids) or creator_id not in member_ids: + raise ValidationError('неверный список участников') + chat_id = uuid.uuid4() + try: + with self.pool.connection() as conn: + row = conn.execute('INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING *', (chat_id,title)).fetchone() + for member in member_ids: + conn.execute('INSERT INTO chat_members VALUES (%s,%s)', (chat_id,member)) + return Chat(**public_row(row), member_ids=sorted(member_ids)) + except ForeignKeyViolation: + raise ValidationError('участник не существует') from None + + def get_chat(self, chat_id): + with self.pool.connection() as conn: + row = conn.execute('SELECT * FROM chats WHERE id=%s', (chat_id,)).fetchone() + if not row: + return None + members = conn.execute('SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (chat_id,)).fetchall() + return Chat(**public_row(row), member_ids=[str(x['user_id']) for x in members]) + + def is_chat_member(self, chat_id, user_id): + """Проверить доступ без загрузки всего чата и списка участников.""" + with self.pool.connection() as conn: + return conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id,user_id)).fetchone() is not None + + def list_chats_for_user(self, user_id, cursor, limit): + with self.pool.connection() as conn: + params = [user_id] + boundary = '' + if cursor is not None: + when, item = self._decode_cursor(cursor, 'chats', user_id) + marker = conn.execute('SELECT c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s', (item,user_id)).fetchone() + if not marker or timestamp(marker['created_at']) != when: + raise ValidationError('некорректный cursor') + boundary = ' AND (c.created_at,c.id) > (%s,%s)' + params += [when,item] + rows = conn.execute('SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s' + boundary + ' ORDER BY c.created_at,c.id LIMIT %s', params+[limit+1]).fetchall() + page = [self.get_chat(str(row['id'])) for row in rows[:limit]] + next_cursor = self._encode_cursor('chats',user_id,page[-1].created_at,page[-1].id) if len(rows)>limit else None + return page,next_cursor + + def create_message(self, chat_id, sender_id, text, key, attachment_ids=None): + attachment_ids = attachment_ids or [] + normalized = {'text': text, 'attachment_ids': attachment_ids} + # Блокировка сериализует один ключ на всех репликах; результат и сообщение атомарны. + with self.pool.connection() as conn: + scope = json.dumps([sender_id, chat_id, key]) + conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (scope,)) + old = conn.execute('SELECT * FROM message_requests WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at>clock_timestamp()', (sender_id,chat_id,key)).fetchone() + if old: + if json.loads(old['body']) != normalized: + raise ConflictError('ключ уже использован с другим сообщением') + row = conn.execute('SELECT * FROM messages WHERE id=%s', (old['message_id'],)).fetchone() + return self.message_model(conn, row) + for attachment_id in sorted(attachment_ids): + attachment = conn.execute('SELECT state FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE', (attachment_id,chat_id)).fetchone() + if not attachment: + raise NotFoundError('вложение не найдено') + if attachment['state'] == 'failed': + raise ConflictError('вложение недоступно') + optimized = os.getenv('OPTIMIZED_WRITES','1') == '1' + if not optimized: + conn.execute('DELETE FROM message_requests WHERE user_id=%s AND chat_id=%s AND key=%s', (sender_id,chat_id,key)) + row = conn.execute('INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *', (uuid.uuid4(),chat_id,sender_id,text)).fetchone() + # Под той же блокировкой UPSERT заменяет просроченный результат без отдельного DELETE. + suffix = ' ON CONFLICT (user_id,chat_id,key) DO UPDATE SET body=EXCLUDED.body,message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at' if optimized else '' + conn.execute("INSERT INTO message_requests VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')"+suffix, (sender_id,chat_id,key,json.dumps(normalized,ensure_ascii=False),row['id'])) + for position, attachment_id in enumerate(attachment_ids): + conn.execute('INSERT INTO message_attachments VALUES (%s,%s,%s)', (row['id'],attachment_id,position)) + result = self.message_model(conn,row) + return result + + @staticmethod + def message_model(conn, row): + ids = conn.execute('SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position', (row['id'],)).fetchall() + return Message(**public_row(row), attachment_ids=[str(x['attachment_id']) for x in ids]) + + def list_messages(self, chat_id, cursor, limit): + with self.pool.connection() as conn: + params = [chat_id] + boundary = '' + if cursor is not None: + when,item = self._decode_cursor(cursor,'messages',chat_id) + marker = conn.execute('SELECT created_at FROM messages WHERE id=%s AND chat_id=%s', (item,chat_id)).fetchone() + if not marker or timestamp(marker['created_at']) != when: + raise ValidationError('некорректный cursor') + boundary = ' AND (created_at,id) > (%s,%s)' + params += [when,item] + rows = conn.execute('SELECT * FROM messages WHERE chat_id=%s' + boundary + ' ORDER BY created_at,id LIMIT %s', params+[limit+1]).fetchall() + page = [self.message_model(conn,row) for row in rows[:limit]] + next_cursor = self._encode_cursor('messages',chat_id,page[-1].created_at,page[-1].id) if len(rows)>limit else None + return page,next_cursor + + def ready(self): + with self.pool.connection() as conn: + conn.execute('SELECT 1 FROM schema_migrations LIMIT 1') + + def acquire_request_slot(self): + with self._lock: + if self._requests >= 100: + return False + self._requests += 1 + return True + + def release_request_slot(self): + with self._lock: + self._requests -= 1 + + def get_stats(self): + with self.pool.connection() as conn: + result = {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n'] for table in ('users','chats','messages')} + result['profile_cache'] = {'hits': profile_cache.hits, 'misses': profile_cache.misses, 'errors': profile_cache.errors, 'ttl_seconds': profile_cache.ttl} + return result + + +storage = PostgresStorage() diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..53ba47d5 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,123 @@ +"""Worker обрабатывает одну доставку за раз и подтверждает после COMMIT.""" + +import hashlib +import json +import logging +import signal +import subprocess +import sys +import tempfile +import threading +from contextlib import closing +from pathlib import Path + +import pika +import psycopg +from botocore.exceptions import BotoCoreError, ClientError +from psycopg_pool import PoolTimeout + +from src import broker, jobs, faults +from src.objects import BUCKET, ObjectConflict, client, put_immutable +from src.storage import storage + + +class PermanentError(Exception): + pass + + +def fetch(s3, key, attachment, destination): + digest = hashlib.sha256() + size = 0 + obj = s3.get_object(Bucket=BUCKET, Key=key) + with closing(obj['Body']) as stream, destination.open('wb') as output: + for chunk in stream.iter_chunks(chunk_size=64 * 1024): + size += len(chunk) + if size > attachment['size_bytes']: + raise PermanentError('staging_integrity') + digest.update(chunk) + output.write(chunk) + if (size, digest.hexdigest()) != (attachment['size_bytes'], attachment['sha256']): + raise PermanentError('staging_integrity') + + +def process(job): + faults.hit('before_effect', job) + attachment = job['attachment'] + s3 = client() + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + original = directory / 'original' + source = attachment['object_key'] if attachment['published_pipeline_version'] else job['staging_key'] + fetch(s3, source, attachment, original) + result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', temp, str(job['pipeline_version'])], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=jobs.JOB_TIMEOUT, check=False) + if result.returncode == 2: + raise PermanentError('invalid_image') + result.check_returncode() + with original.open('rb') as stream: + put_immutable(s3, attachment['object_key'], stream, attachment['size_bytes'], + attachment['content_type'], attachment['sha256']) + variants = [] + faults.hit('after_original', job) + for name, mime, width, height in json.loads((directory / 'manifest.json').read_text()): + key = f"variants/{attachment['id']}/v{job['pipeline_version']}/{name}" + output = directory / name + size = output.stat().st_size + with output.open('rb') as stream: + digest = hashlib.file_digest(stream, 'sha256').hexdigest() + stream.seek(0) + put_immutable(s3, key, stream, size, mime, digest) + variants.append((name, key, mime, size, digest, width, height)) + faults.hit('after_variants', job) + jobs.complete(job, variants) + faults.hit('after_commit', job) + + +def handle(channel, delivery, body): + try: + message = jobs.validate(json.loads(body)) + except (ValueError, TypeError, AttributeError): + logging.warning('invalid broker job') + channel.basic_reject(delivery.delivery_tag, requeue=False) + return + + job = jobs.claim(message) + if job: + try: + process(job) + except PermanentError as exc: + jobs.fail(job, str(exc), permanent=True) + except ObjectConflict: + jobs.fail(job, 'object_conflict', permanent=True) + except (BotoCoreError, ClientError, OSError, subprocess.SubprocessError) as exc: + logging.warning('processing retry job=%s cause=%s', job['id'], type(exc).__name__) + jobs.fail(job, 'processing_timeout' if isinstance(exc, subprocess.TimeoutExpired) else 'processing_failed') + channel.basic_ack(delivery.delivery_tag) + + +def main(): + logging.basicConfig(level=logging.WARNING) + stopping = threading.Event() + for sig in (signal.SIGINT, signal.SIGTERM): + signal.signal(sig, lambda *_: stopping.set()) + storage.pool.open(wait=True) + try: + while not stopping.is_set(): + try: + with broker.channel(heartbeat=jobs.LEASE_SECONDS + 15) as (_, channel): + channel.basic_qos(prefetch_count=1) + for delivery, _, body in channel.consume(broker.QUEUE, inactivity_timeout=0.5): + if stopping.is_set(): + break + if delivery: + handle(channel, delivery, body) + except (pika.exceptions.AMQPError, OSError, psycopg.Error, PoolTimeout) as exc: + logging.warning('worker temporarily unavailable: %s', type(exc).__name__) + stopping.wait(1) + finally: + storage.pool.close() + + +if __name__ == '__main__': + main() diff --git a/student-tests/load_messages.py b/student-tests/load_messages.py new file mode 100644 index 00000000..ec772c6a --- /dev/null +++ b/student-tests/load_messages.py @@ -0,0 +1,22 @@ +"""Публичная нагрузка сообщений с отдельными cgroup CPU/memory snapshots.""" +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +command = [sys.executable, '-X', 'utf8', 'tests/load.py', '--base-url', 'http://127.0.0.1:8085', + '--duration', '30', '--concurrency', '4', '--output', 'evidence/load-messages.json'] +with (ROOT / 'evidence/load-messages-run.txt').open('w', encoding='utf-8') as output: + process = subprocess.Popen(command, cwd=ROOT, stdout=output, stderr=subprocess.STDOUT) + samples = [] + start = time.monotonic() + while process.poll() is None: + ids = subprocess.check_output(['docker', 'compose', '-p', 'lyakhonov-lab5', 'ps', '-q'], cwd=ROOT, text=True).split() + stats = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids], text=True) + samples.append(dict(at_seconds=round(time.monotonic()-start, 3), containers=[json.loads(x) for x in stats.splitlines()])) + time.sleep(2) + code = process.wait() +(ROOT / 'evidence/load-messages-resources.json').write_text(json.dumps(dict(command=command, exit_code=code, samples=samples), indent=2), encoding='utf-8') +raise SystemExit(code) diff --git a/student-tests/load_pipeline.py b/student-tests/load_pipeline.py new file mode 100644 index 00000000..8fc3e411 --- /dev/null +++ b/student-tests/load_pipeline.py @@ -0,0 +1,94 @@ +"""Closed-loop: полный путь upload -> ready; ограничение генератора явно в отчёте.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path +import random +import struct +import subprocess +import sys +import threading +import time +import zlib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def png(width, height): + def chunk(name, data): + return struct.pack('>I', len(data)) + name + data + struct.pack('>I', zlib.crc32(name + data)) + rng = random.Random(2301) + data = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height)) + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(data)) + chunk(b'IEND', b'')) + + +def percentile(values, q): + return round(sorted(values)[min(len(values)-1, int((len(values)-1)*q))], 4) if values else None + + +def main(args): + client = Client(args.base_url) + alice, _, _, chat = client.bootstrap() + profiles = [('320x200', (ROOT / 'tests/fixtures/sample.png').read_bytes()), ('1024x768', png(1024, 768))] + rows = [] + for profile, raw in profiles: + for repeat in range(args.repeats): + samples = [] + failures = [] + stats = [] + stop = threading.Event() + def sample(): + while not stop.is_set(): + ids = subprocess.check_output(['docker', 'compose', '-p', args.project, 'ps', '-q'], text=True).split() + result = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids], text=True) + stats.append(dict(at_seconds=round(time.monotonic()-start, 3), containers=[json.loads(line) for line in result.splitlines()])) + stop.wait(2) + def one(): + c = Client(args.base_url) + begin = time.monotonic() + att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png', 202, + raw=raw, actor=alice, headers={'Content-Type': 'image/png'}, schema='Attachment')['json'] + accepted = time.monotonic() + ready = c.wait_ready(alice, att) + check(len(ready['variants']) == 3 and ready['sha256'] == hashlib.sha256(raw).hexdigest(), 'bad load result') + return dict(upload_seconds=accepted-begin, ready_seconds=time.monotonic()-begin) + # Один завершённый прогрев на профиль/повтор, не входит в измерение. + one() + start = time.monotonic() + sampler = threading.Thread(target=sample) + sampler.start() + def loop(_): + while time.monotonic()-start < args.duration: + try: + samples.append(one()) + except Exception as exc: + failures.append(type(exc).__name__ + ': ' + str(exc)) + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + list(pool.map(loop, range(args.concurrency))) + elapsed = time.monotonic()-start + stop.set() + sampler.join() + times = [x['ready_seconds'] for x in samples] + row = dict(profile=profile, bytes_per_image=len(raw), repeat=repeat+1, concurrency=args.concurrency, + requested_duration=args.duration, elapsed_seconds=round(elapsed, 3), warmup_completed=1, + completed=len(samples), successful_rps=round(len(samples)/elapsed, 3), failures=failures, + error_fraction=len(failures)/max(1,len(samples)+len(failures)), + p50=percentile(times,.5), p95=percentile(times,.95), p99=percentile(times,.99), + upload_p95=percentile([x['upload_seconds'] for x in samples],.95), stats=stats) + rows.append(row) + (ROOT / 'evidence/load-pipeline.json').write_text(json.dumps(rows, indent=2), encoding='utf-8') + print(json.dumps({k:v for k,v in row.items() if k != 'stats'}), flush=True) + + +if __name__ == '__main__': + p = argparse.ArgumentParser() + p.add_argument('--base-url', default='http://127.0.0.1:8085') + p.add_argument('--project', default='lyakhonov-lab5') + p.add_argument('--duration', type=int, default=20) + p.add_argument('--concurrency', type=int, default=2) + p.add_argument('--repeats', type=int, default=3) + main(p.parse_args()) diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100644 index 00000000..ee1f46f9 --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,243 @@ +"""Реальные fault-injection и инварианты. Запускать только на своём lab5 стенде.""" +import argparse +import base64 +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import os +from pathlib import Path +import struct +import subprocess +import sys +import time +import zlib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +class Experiment: + def __init__(self, args): + self.args = args + self.compose = ['docker', 'compose', '-p', args.project] + self.c = Client(args.base_url) + self.alice, self.bob, self.eve, self.chat = self.c.bootstrap() + path = ROOT / 'evidence/pipeline.json' + self.rows = json.loads(path.read_text(encoding='utf-8')) if args.resume and path.exists() else [] + + def command(self, *args): + return subprocess.check_output([*self.compose, *args], cwd=ROOT, text=True, encoding='utf-8', stderr=subprocess.PIPE) + + def probe(self, action, **kwargs): + script = 'ARGS=' + repr(dict(action=action, **kwargs)) + '\n' + (ROOT / 'student-tests/probe.py').read_text(encoding='utf-8') + invocation = ([*self.compose, 'run', '--rm', '--no-deps', '-T', 'inspector', 'python', '-'] + if action == 'objects' else [*self.compose, 'exec', '-T', '--index', '1', 'api', 'python', '-']) + result = subprocess.run(invocation, + input=script, text=True, encoding='utf-8', capture_output=True, cwd=ROOT) + check(result.returncode == 0, 'probe failed: ' + result.stderr) + return json.loads(result.stdout) + + def sql(self, query, *params): + return self.probe('sql', query=query, params=list(params)) + + def job(self, att, version=1): + return self.sql('SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s', att['id'], version)[0] + + @staticmethod + def envelope(job): + from datetime import datetime, timezone + created = datetime.fromisoformat(job['created_at']).astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + return dict(schema_version=1, type='attachment.process.v1', job_id=job['id'], attachment_id=job['attachment_id'], + pipeline_version=job['pipeline_version'], attempt=job['attempt'], created_at=created) + + def upload(self): + return self.c.attachment(self.alice, self.chat)[0] + + def arm(self, job, point, action='crash'): + self.sql('INSERT INTO experiment_faults(job_id,point,action) VALUES (%s,%s,%s)', job['id'], point, action) + + def hit(self, job, point): + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + row = self.sql('SELECT * FROM experiment_faults WHERE job_id=%s AND point=%s', job['id'], point)[0] + if row['hit_at']: + return row + time.sleep(.3) + raise AssertionError('fault not hit: ' + point) + + def snapshot(self): + ids = self.command('ps', '-a', '-q').split() + raw = subprocess.check_output(['docker', 'inspect', *ids], text=True, encoding='utf-8') + return [dict(id=x['Id'], name=x['Name'], started_at=x['State']['StartedAt'], + running=x['State']['Running'], restart_count=x['RestartCount'], exit_code=x['State']['ExitCode']) for x in json.loads(raw)] + + def invariant(self, att, version=1): + ready = self.c.wait_ready(self.alice, att, wait_seconds=120) + check(len(ready['variants']) == 3, 'three variants required') + row = self.sql('SELECT published_pipeline_version,target_pipeline_version FROM attachments WHERE id=%s', att['id'])[0] + check(row['published_pipeline_version'] == version, 'wrong published version') + variants = self.sql('SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s', att['id'], version) + check(len(variants) == 3, 'duplicate/missing manifests') + digest = {} + for name in ('original', 'thumbnail', 'crop', 'optimized'): + response = self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + name, actor=self.alice) + sha = hashlib.sha256(response['bytes']).hexdigest() + expected = att['sha256'] if name == 'original' else next(v['sha256'] for v in variants if v['name'] == name) + check(sha == expected, 'download checksum mismatch') + digest[name] = sha + return digest + + def record(self, name, started, **details): + self.rows.append(dict(scenario=name, seconds=round(time.monotonic() - started, 3), **details)) + (ROOT / 'evidence/pipeline.json').write_text(json.dumps(self.rows, ensure_ascii=False, indent=2), encoding='utf-8') + print('PASS:', name, flush=True) + + def crashes(self): + for repeat in range(self.args.repeats): + for point in ('before_publish', 'after_publish', 'before_effect', 'after_original', 'after_variants', 'after_commit'): + start = time.monotonic() + service = 'dispatcher' if 'publish' in point else 'worker' + self.command('stop', '-t', '2', service) + att = self.upload() + job = self.job(att) + self.arm(job, point) + before = self.snapshot() + self.command('start', service) + hit = self.hit(job, point) + self.invariant(att) + self.record(point, start, repeat=repeat + 1, attachment_id=att['id'], fault=hit, + job_after=self.job(att), before=before, after=self.snapshot()) + + def durable(self): + start = time.monotonic() + self.command('stop', '-t', '2', 'worker') + att = self.upload() + self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=self.alice) + self.probe('cleanup') + check(any(obj['key'].startswith('staging/') for obj in self.probe('objects', id=att['id'])), 'active staging deleted') + self.c.message(self.alice, self.chat, '', attachments=[att['id']]) + time.sleep(2) + before = self.snapshot() + self.command('restart', '-t', '2', 'api', 'broker') + self.command('start', 'worker') + self.invariant(att) + self.record('accepted_survives_api_broker_restart', start, attachment_id=att['id'], before=before, after=self.snapshot()) + + def versions(self): + for repeat in range(self.args.repeats): + start = time.monotonic() + self.command('stop', '-t', '2', 'worker') + att = self.upload() + old = self.job(att) + self.arm(old, 'after_variants', 'pause') + self.command('start', 'worker') + self.hit(old, 'after_variants') + self.probe('upgrade', id=att['id']) + new = self.job(att, 2) + # Новая работа, затем старая, затем 24 чередующихся дубля. + self.probe('publish', messages=[self.envelope(new), self.envelope(old)] * 13) + first = self.invariant(att, 2) + objects = self.probe('objects', id=att['id']) + self.sql('UPDATE experiment_faults SET released=true WHERE job_id=%s', old['id']) + time.sleep(2) + check(self.invariant(att, 2) == first, 'stale worker changed new result') + self.probe('publish', messages=[self.envelope(old), self.envelope(new)] * 13) + time.sleep(2) + check(self.invariant(att, 2) == first, 'reorder changed new result') + current = self.probe('objects', id=att['id']) + check([x for x in current if not x['key'].startswith('staging/')] == + [x for x in objects if not x['key'].startswith('staging/')], 'duplicate wrote more objects') + check(self.probe('replay', id=att['id']) is None, 'ready replay must no-op') + self.record('stale_running_worker_and_reordered_duplicates', start, repeat=repeat + 1, + attachment_id=att['id'], before_digest=first, after_digest=self.invariant(att, 2), + versions=self.sql('SELECT pipeline_version,state,attempt FROM attachment_jobs WHERE attachment_id=%s ORDER BY pipeline_version', att['id']), objects=current) + start = time.monotonic() + att = self.upload() + old_digest = self.invariant(att) + self.probe('cleanup') + self.probe('upgrade', id=att['id']) + deadline = time.monotonic() + 60 + while self.job(att, 2)['state'] != 'ready': + check(time.monotonic() < deadline, 'ready upgrade timeout') + time.sleep(.3) + new_digest = self.invariant(att, 2) + check(new_digest['original'] == old_digest['original'], 'upgrade changed original') + check(new_digest['thumbnail'] != old_digest['thumbnail'], 'versions must have a real different effect') + self.record('upgrade_ready_after_staging_cleanup', start, repeat=repeat + 1, attachment_id=att['id'], before_digest=old_digest, after_digest=new_digest) + + def errors(self): + for kind in ('permanent', 'transient'): + start = time.monotonic() + self.command('stop', '-t', '2', 'worker') + att = self.upload() + if kind == 'permanent': + self.probe('corrupt_staging', id=att['id']) + else: + self.probe('remove_staging', id=att['id']) + self.command('start', 'worker') + observed = [] + deadline = time.monotonic() + 120 + while True: + job = self.job(att) + observed.append(dict(attempt=job['attempt'], state=job['state'], next_publish_at=job['next_publish_at'], last_error=job['last_error'])) + if job['state'] == 'failed': + break + check(time.monotonic() < deadline, 'failed registry timeout') + time.sleep(.3) + check(job['attempt'] == (1 if kind == 'permanent' else job['attempt_limit']), 'wrong retry budget') + self.c.call('POST', '/api/v1/chats/' + self.chat['id'] + '/messages', 409, actor=self.alice, + headers={'Idempotency-Key': os.urandom(12).hex()}, body={'text': '', 'attachment_ids': [att['id']]}) + self.probe('restore_staging', id=att['id'], data=base64.b64encode((ROOT / 'tests/fixtures/sample.png').read_bytes()).decode()) + check(self.probe('replay', id=att['id']) is not None, 'failed replay missing') + self.invariant(att) + self.record(kind + '_error_and_explicit_replay', start, attachment_id=att['id'], attempts=observed, + audit=self.sql('SELECT attempt,reason FROM attachment_replays WHERE job_id=%s', job['id'])) + + def negatives(self): + start = time.monotonic() + path = '/api/v1/chats/' + self.chat['id'] + '/attachments?filename=x.png' + def upload(raw, expected, path=path, actor=None, mime='image/png'): + self.c.call('POST', path, expected, actor=actor or self.alice, raw=raw, headers={'Content-Type': mime}) + upload(b'bad', 415) + upload(b'x' * (10 * 1024 * 1024 + 1), 413) + raw = (ROOT / 'tests/fixtures/sample.png').read_bytes() + upload(raw, 415, mime='image/jpeg') + upload(raw, 400, path=path.replace('x.png', '..%2Fx.png')) + upload(raw, 404, actor=self.eve) + # Валидный PNG header >40 MP: отклонение до декодирования большого bitmap. + def chunk(name, data): + return struct.pack('>I', len(data)) + name + data + struct.pack('>I', zlib.crc32(name + data)) + huge = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 10000, 10000, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b'')) + chunk(b'IEND', b'') + upload(huge, 413) + att = self.upload() + self.invariant(att) + check(self.probe('anonymous', id=att['id']) == 403, 'bucket allows anonymous reads') + self.c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=self.eve) + self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=self.eve) + key = os.urandom(16).hex() + with ThreadPoolExecutor(max_workers=12) as pool: + responses = list(pool.map(lambda _: self.c.message(self.alice, self.chat, 'concurrent', key=key)['json']['id'], range(24))) + check(len(set(responses)) == 1, 'message idempotency violated') + self.record('negative_upload_acl_and_24_concurrent_messages', start, checks=9, message_ids=len(set(responses))) + + def main(self): + try: + if not self.args.resume: + self.negatives() + self.durable() + self.crashes() + self.versions() + self.errors() + finally: + self.sql('UPDATE experiment_faults SET released=true WHERE NOT released') + self.command('start', 'api', 'broker', 'dispatcher', 'worker') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--project', default='lyakhonov-lab5') + parser.add_argument('--base-url', default='http://127.0.0.1:8085') + parser.add_argument('--repeats', type=int, default=3) + parser.add_argument('--resume', action='store_true', help='Сохранить уже выполненные crash-результаты; выполнить version/error suite') + Experiment(parser.parse_args()).main() diff --git a/student-tests/probe.py b/student-tests/probe.py new file mode 100644 index 00000000..48674199 --- /dev/null +++ b/student-tests/probe.py @@ -0,0 +1,66 @@ +"""Исполняется внутри API через stdin; наружу возвращает только безопасные данные.""" +import json +import sys +from uuid import UUID + +from src import broker, jobs +from src.objects import BUCKET, client +from src.storage import storage + + +def run(args): + action = args['action'] + if action == 'anonymous': + import boto3 + from botocore import UNSIGNED + from botocore.config import Config + from botocore.exceptions import ClientError + s3 = boto3.client('s3', endpoint_url='http://s3:9000', config=Config(signature_version=UNSIGNED)) + try: + s3.get_object(Bucket=BUCKET, Key=f"originals/{args['id']}") + except ClientError as exc: + return exc.response['ResponseMetadata']['HTTPStatusCode'] + return 200 + if action == 'sql': + with storage.pool.connection() as conn: + cursor = conn.execute(args['query'], args.get('params', [])) + return cursor.fetchall() if cursor.description else [] + if action == 'publish': + with broker.channel() as (_, channel): + channel.confirm_delivery() + for message in args['messages']: + broker.publish(channel, message) + return True + if action == 'upgrade': + return jobs.upgrade(UUID(args['id']), args.get('version', 2)) + if action == 'replay': + return jobs.replay(UUID(args['id'])) + if action == 'cleanup': + from src.staging import cleanup + return cleanup(client(), 60) + if action == 'objects': + s3 = client() + result = [] + for prefix in (f"originals/{args['id']}", f"variants/{args['id']}/", f"staging/{args['id']}"): + for page in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=prefix): + result.extend({'key': row['Key'], 'etag': row['ETag'], 'size': row['Size']} + for row in page.get('Contents', [])) + return result + if action == 'remove_staging': + client().delete_object(Bucket=BUCKET, Key=f"staging/{args['id']}") + return True + if action == 'corrupt_staging': + client().put_object(Bucket=BUCKET, Key=f"staging/{args['id']}", Body=b'broken') + return True + if action == 'restore_staging': + import base64 + client().put_object(Bucket=BUCKET, Key=f"staging/{args['id']}", Body=base64.b64decode(args['data'])) + return True + raise ValueError('unknown probe action') + + +storage.pool.open(wait=True) +try: + print(json.dumps(run(ARGS), default=str)) +finally: + storage.pool.close() diff --git a/student-tests/resilience_driver.py b/student-tests/resilience_driver.py new file mode 100644 index 00000000..62fc51ae --- /dev/null +++ b/student-tests/resilience_driver.py @@ -0,0 +1,82 @@ +"""Публичные resilience и отдельный принудительный отказ каждой API-реплики.""" +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check + +COMPOSE = ['docker', 'compose', '-p', 'lyakhonov-lab5'] + + +def docker(*args): + return subprocess.check_output(['docker', *args], cwd=ROOT, text=True, encoding='utf-8') + + +def snapshot(): + ids = docker('compose', '-p', 'lyakhonov-lab5', 'ps', '-a', '-q').split() + raw = json.loads(docker('inspect', *ids)) + return [dict(id=x['Id'], name=x['Name'], started_at=x['State']['StartedAt'], running=x['State']['Running']) for x in raw] + + +def public(mode): + before = snapshot() + script = ROOT / 'scripts' / (('restart' if mode == 'persistence' else 'stop-one') + ('.cmd' if sys.platform == 'win32' else '.sh')) + result = subprocess.run([sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', mode, + '--base-url', 'http://127.0.0.1:8085', '--action-script', str(script)], + cwd=ROOT, text=True, encoding='utf-8', capture_output=True) + after = snapshot() + (ROOT / f'evidence/{mode}.json').write_text(json.dumps(dict(command=result.args, exit_code=result.returncode, + output=result.stdout + result.stderr, before=before, after=after), ensure_ascii=False, indent=2), encoding='utf-8') + print(result.stdout, flush=True) + check(result.returncode == 0, result.stderr) + subprocess.check_call([*COMPOSE, 'start', 'api'], cwd=ROOT) + + +def targeted(): + c = Client('http://127.0.0.1:8085', timeout=2) + alice, _, _, chat = c.bootstrap() + ids = docker('compose', '-p', 'lyakhonov-lab5', 'ps', '-q', 'api').split() + evidence = [] + for index, target in enumerate(ids): + before = snapshot() + responses, errors = [], [] + first = None + begin = time.monotonic() + subprocess.check_call(['docker', 'stop', '-t', '2', target], stdout=subprocess.DEVNULL) + try: + while time.monotonic() - begin < 15: + key = f'explicit-{index}-{len(responses)}' + try: + response = c.message(alice, chat, 'replica failure', key=key) + first = first or time.monotonic()-begin + responses.append(response['json']) + except Failure as exc: + errors.append(str(exc)) + time.sleep(.05) + check(first is not None and first <= 10, 'slow API failover') + page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice, schema='MessagePage')['json'] + messages = list(page['items']) + while page['next_cursor']: + from urllib.parse import quote + page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe=''), actor=alice, schema='MessagePage')['json'] + messages.extend(page['items']) + found = {m['id'] for m in messages} + check(all(m['id'] in found for m in responses), 'acknowledged message lost') + check(len(found) == len(messages), 'duplicate messages') + evidence.append(dict(target=target, before=before, after=snapshot(), successes=len(responses), + errors=errors, recovery_seconds=round(first, 3))) + finally: + subprocess.check_call(['docker', 'start', target], stdout=subprocess.DEVNULL) + time.sleep(3) + (ROOT / 'evidence/failover-each.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2), encoding='utf-8') + print('PASS: both real API containers independently stopped and restored') + + +if __name__ == '__main__': + public('persistence') + public('failover') + targeted() diff --git a/student-tests/test_images.py b/student-tests/test_images.py new file mode 100644 index 00000000..eadae04d --- /dev/null +++ b/student-tests/test_images.py @@ -0,0 +1,53 @@ +import hashlib +import tempfile +import unittest +from pathlib import Path + +from PIL import Image + +from src.image_pipeline import render + +ROOT = Path(__file__).resolve().parents[1] + + +class Images(unittest.TestCase): + def test_public_fixture_and_repeat(self): + fixture = ROOT / 'tests' / 'fixtures' / 'sample.png' + original = hashlib.sha256(fixture.read_bytes()).hexdigest() + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) + expected = { + 'thumbnail': ('JPEG', (256, 160)), + 'crop': ('JPEG', (128, 128)), + 'optimized': ('WEBP', (320, 200)), + } + first = render(fixture, target) + self.assertEqual(len(first), len(expected)) + digests = {name: hashlib.sha256((target / name).read_bytes()).hexdigest() for name in expected} + self.assertEqual(first, render(fixture, target)) + for name, properties in expected.items(): + with Image.open(target / name) as image: + self.assertEqual((image.format, image.size), properties) + self.assertEqual(hashlib.sha256((target / name).read_bytes()).hexdigest(), digests[name]) + self.assertEqual(hashlib.sha256(fixture.read_bytes()).hexdigest(), original) + + def test_orientation_alpha_and_small_image(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) + source = target / 'source.png' + image = Image.new('RGBA', (35, 15), (255, 0, 0, 0)) + image.save(source, exif=self.orientation()) + render(source, target) + with Image.open(target / 'thumbnail') as thumbnail: + self.assertEqual(thumbnail.size, (15, 35)) + self.assertEqual(thumbnail.getpixel((4, 4)), (255, 255, 255)) + self.assertFalse(thumbnail.getexif()) + with Image.open(target / 'optimized') as optimized: + self.assertEqual(optimized.size, (15, 35)) + self.assertFalse(optimized.getexif()) + + @staticmethod + def orientation(): + exif = Image.Exif() + exif[274] = 6 + return exif