From 8dad129edfdaee5e03f7d099a6f398094e72b44e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D0=BD=D0=B5=D0=BA?= <88394552+Lunatuk@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:53:24 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D1=82=D1=8C=20=D0=BB=D0=B0=D0=B1=D0=BE=D1=80=D0=B0?= =?UTF-8?q?=D1=82=D0=BE=D1=80=D0=BD=D1=83=D1=8E=206=20=D0=BD=D0=B0=205:=20?= =?UTF-8?q?SLO=20=D0=B8=20=D0=BE=D0=B1=D0=BD=D0=B0=D1=80=D1=83=D0=B6=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=B8=D0=BD=D1=86=D0=B8=D0=B4=D0=B5=D0=BD?= =?UTF-8?q?=D1=82=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 31 +- .env.example | 12 +- .github/pull_request_template.lab5.md | 58 + .github/pull_request_template.md | 67 +- .github/workflows/application.yml | 24 + Dockerfile | 28 + REPORT.lab5.md | 184 + REPORT.md | 232 +- compose.yaml | 289 + docs/runbook.md | 91 + docs/schema.md | 40 + evidence/docker-checks.txt | 66 + evidence/lab3-checks.json | 1209 + evidence/lab3-checks.txt | 403 + evidence/lab4-checks.json | 2269 ++ evidence/lab4-checks.txt | 533 + evidence/lab5-checks.json | 502 + evidence/lab5-checks.txt | 77 + evidence/lab5-restart.json | 140 + evidence/lab5-restart.txt | 4 + evidence/lab6-api-checks.txt | 51 + evidence/lab6-checks.json | 102 + evidence/lab6-checks.txt | 102 + evidence/lab6-correlated-logs.json | 306 + evidence/lab6-dashboard-checks.json | 100 + evidence/lab6-db-incident-logs.json | 191 + evidence/lab6-db-incident-trace.json | 275 + evidence/lab6-lab5-regression.json | 795 + evidence/lab6-lab5-regression.txt | 77 + evidence/lab6-promtool.txt | 1 + evidence/lab6-restart.json | 206 + evidence/lab6-restart.txt | 4 + evidence/lab6-runtime.json | 322 + evidence/lab6-sdk-buffers.json | 666 + evidence/lab6-slo-research.json | 20389 ++++++++++++++++ evidence/lab6-slo-research.txt | 246 + evidence/lab6-slo-series.json | 8354 +++++++ evidence/lab6-source-manifest.json | 130 + evidence/lab6-telemetry-persistence.json | 12 + evidence/lab6-tempo-search.json | 2167 ++ evidence/lab6-trace-retry.json | 1701 ++ evidence/lab6-worker-incident-logs.json | 248 + evidence/lab6-worker-incident-trace.json | 1161 + evidence/postgres-pause.json | 76 + evidence/query-plans.json | 2621 ++ evidence/research.json | 1114 + evidence/research.txt | 32 + infra/grafana/dashboards/messenger.json | 618 + .../provisioning/dashboards/provider.yaml | 9 + .../provisioning/datasources/sources.yaml | 35 + infra/haproxy.cfg | 40 + infra/otel/alerts.yaml | 113 + infra/otel/collector.yaml | 74 + infra/otel/loki.yaml | 37 + infra/otel/prometheus.yaml | 15 + infra/otel/tempo.yaml | 20 + migrations/001_initial.sql | 30 + migrations/002_message_idempotency.sql | 10 + migrations/003_attachments.sql | 34 + migrations/004_attachment_jobs.sql | 42 + migrations/005_observability.sql | 9 + requirements.txt | 50 + research/README.lab2.md | 12 + research/README.md | 37 + research/__init__.py | 0 research/benchmark.py | 168 + research/observations.py | 111 + research/run.py | 99 + research/slo.py | 207 + scripts/build.sh | 6 + scripts/check-docker.sh | 11 + scripts/check-lab6.sh | 29 + scripts/check.sh | 17 + scripts/demo.sh | 28 + scripts/init_local.py | 62 + scripts/migrate.sh | 6 + scripts/replicas.py | 90 + scripts/research-lab2.sh | 7 + scripts/research.sh | 7 + scripts/restart.sh | 7 + scripts/rolling-update.sh | 7 + scripts/run-local.sh | 5 + scripts/start-worker.sh | 5 + scripts/start.sh | 8 + scripts/stop-one.sh | 5 + scripts/stop-worker.sh | 5 + scripts/stop.sh | 6 + scripts/wait-observability.py | 26 + scripts/wait-ready.sh | 5 + src/__init__.py | 0 src/__main__.py | 16 + src/app.py | 222 + src/attachments.py | 154 + src/auth.py | 95 + src/bootstrap_s3.py | 38 + src/broker.py | 46 + src/cleanup.py | 87 + src/config.py | 20 + src/db.py | 63 + src/errors.py | 27 + src/faults.py | 31 + src/healthcheck.py | 15 + src/image_pipeline.py | 70 + src/job_admin.py | 29 + src/jobs.py | 103 + src/migrate.py | 48 + src/models.py | 56 + src/object_store.py | 61 + src/pagination.py | 39 + src/passwords.py | 22 + src/publisher.py | 46 + src/storage.py | 156 + src/telemetry.py | 148 + src/worker.py | 227 + student-tests/attachment_checks.py | 275 + student-tests/docker_lifecycle.py | 319 + student-tests/helpers.py | 42 + student-tests/lab5_pipeline.py | 325 + student-tests/lab6_observability.py | 117 + student-tests/postgres_pause.py | 145 + student-tests/promtool-slo.yaml | 199 + student-tests/test_api.py | 164 + student-tests/test_attachments.py | 135 + student-tests/test_concurrency.py | 47 + student-tests/test_idempotency.py | 62 + student-tests/test_sessions.py | 51 + student-tests/verify_restart.py | 37 + 127 files changed, 53496 insertions(+), 61 deletions(-) create mode 100644 .github/pull_request_template.lab5.md create mode 100644 .github/workflows/application.yml create mode 100644 Dockerfile create mode 100644 REPORT.lab5.md create mode 100644 compose.yaml create mode 100644 docs/runbook.md create mode 100644 docs/schema.md create mode 100644 evidence/docker-checks.txt create mode 100644 evidence/lab3-checks.json create mode 100644 evidence/lab3-checks.txt create mode 100644 evidence/lab4-checks.json create mode 100644 evidence/lab4-checks.txt create mode 100644 evidence/lab5-checks.json create mode 100644 evidence/lab5-checks.txt create mode 100644 evidence/lab5-restart.json create mode 100644 evidence/lab5-restart.txt create mode 100644 evidence/lab6-api-checks.txt create mode 100644 evidence/lab6-checks.json create mode 100644 evidence/lab6-checks.txt create mode 100644 evidence/lab6-correlated-logs.json create mode 100644 evidence/lab6-dashboard-checks.json create mode 100644 evidence/lab6-db-incident-logs.json create mode 100644 evidence/lab6-db-incident-trace.json create mode 100644 evidence/lab6-lab5-regression.json create mode 100644 evidence/lab6-lab5-regression.txt create mode 100644 evidence/lab6-promtool.txt create mode 100644 evidence/lab6-restart.json create mode 100644 evidence/lab6-restart.txt create mode 100644 evidence/lab6-runtime.json create mode 100644 evidence/lab6-sdk-buffers.json create mode 100644 evidence/lab6-slo-research.json create mode 100644 evidence/lab6-slo-research.txt create mode 100644 evidence/lab6-slo-series.json create mode 100644 evidence/lab6-source-manifest.json create mode 100644 evidence/lab6-telemetry-persistence.json create mode 100644 evidence/lab6-tempo-search.json create mode 100644 evidence/lab6-trace-retry.json create mode 100644 evidence/lab6-worker-incident-logs.json create mode 100644 evidence/lab6-worker-incident-trace.json create mode 100644 evidence/postgres-pause.json create mode 100644 evidence/query-plans.json create mode 100644 evidence/research.json create mode 100644 evidence/research.txt create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/provider.yaml create mode 100644 infra/grafana/provisioning/datasources/sources.yaml create mode 100644 infra/haproxy.cfg create mode 100644 infra/otel/alerts.yaml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/otel/loki.yaml create mode 100644 infra/otel/prometheus.yaml create mode 100644 infra/otel/tempo.yaml create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_message_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_attachment_jobs.sql create mode 100644 migrations/005_observability.sql create mode 100644 requirements.txt create mode 100644 research/README.lab2.md create mode 100644 research/README.md create mode 100644 research/__init__.py create mode 100644 research/benchmark.py create mode 100644 research/observations.py create mode 100644 research/run.py create mode 100644 research/slo.py create mode 100755 scripts/build.sh create mode 100755 scripts/check-docker.sh create mode 100755 scripts/check-lab6.sh create mode 100755 scripts/check.sh create mode 100755 scripts/demo.sh create mode 100644 scripts/init_local.py create mode 100755 scripts/migrate.sh create mode 100644 scripts/replicas.py create mode 100755 scripts/research-lab2.sh create mode 100755 scripts/research.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/run-local.sh create mode 100755 scripts/start-worker.sh create mode 100755 scripts/start.sh create mode 100755 scripts/stop-one.sh create mode 100755 scripts/stop-worker.sh create mode 100755 scripts/stop.sh create mode 100644 scripts/wait-observability.py create mode 100755 scripts/wait-ready.sh create mode 100644 src/__init__.py create mode 100644 src/__main__.py create mode 100644 src/app.py create mode 100644 src/attachments.py create mode 100644 src/auth.py create mode 100644 src/bootstrap_s3.py create mode 100644 src/broker.py create mode 100644 src/cleanup.py create mode 100644 src/config.py create mode 100644 src/db.py create mode 100644 src/errors.py create mode 100644 src/faults.py create mode 100644 src/healthcheck.py create mode 100644 src/image_pipeline.py create mode 100644 src/job_admin.py create mode 100644 src/jobs.py create mode 100644 src/migrate.py create mode 100644 src/models.py create mode 100644 src/object_store.py create mode 100644 src/pagination.py create mode 100644 src/passwords.py create mode 100644 src/publisher.py create mode 100644 src/storage.py create mode 100644 src/telemetry.py create mode 100644 src/worker.py create mode 100644 student-tests/attachment_checks.py create mode 100644 student-tests/docker_lifecycle.py create mode 100644 student-tests/helpers.py create mode 100644 student-tests/lab5_pipeline.py create mode 100644 student-tests/lab6_observability.py create mode 100644 student-tests/postgres_pause.py create mode 100644 student-tests/promtool-slo.yaml create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_concurrency.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_sessions.py create mode 100644 student-tests/verify_restart.py diff --git a/.dockerignore b/.dockerignore index bc16e55..40d419e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,20 +1,11 @@ -.git -.github -.env -.env.* -!.env.example -secrets -certs -volumes -data -backups -*.key -*.pem -*.p12 -*.pfx -*.jks -.venv -__pycache__ -node_modules -evidence -.DS_Store +* +!Dockerfile +!requirements.txt +!src/ +!src/** +!migrations/ +!migrations/** +!research/ +!research/** +**/__pycache__/ +**/*.py[cod] diff --git a/.env.example b/.env.example index 2b32655..3cc76f2 100644 --- a/.env.example +++ b/.env.example @@ -3,11 +3,12 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +HOST_PORT=8080 SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# Shared sessions carried over from lab4 use Valkey. +SESSION_STORE=valkey CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 @@ -18,13 +19,20 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +UPLOAD_CLEANUP_GRACE_SECONDS=3600 BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 +JOB_LEASE_SECONDS=90 +JOB_BACKOFF_SECONDS=2 +WORKER_TASK_MEMORY_MB=640 OTEL_SERVICE_NAME=messenger-api # Worker must set a distinct service.name. OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +GRAFANA_PORT=3000 +OTEL_SDK_DISABLED=false +OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED=true diff --git a/.github/pull_request_template.lab5.md b/.github/pull_request_template.lab5.md new file mode 100644 index 0000000..59317cf --- /dev/null +++ b/.github/pull_request_template.lab5.md @@ -0,0 +1,58 @@ +## Лабораторная 5 + +Автор / группа: Савельев Иван / БВТ2304 +Целевая оценка (3/4/5): **4** — выполнены требования уровней 3 и 4. +Трек 5, если заявлен: не заявлен. + +### Что реализовано и что перенесено + +Из lab4 перенесены API мессенджера, авторизация и сессии, ACL, идемпотентность сообщений, миграции, приватный S3, две API-реплики за HAProxy, Docker-конфигурация и собственные тесты. + +- Загрузка возвращает `202 queued` после устойчивого сохранения оригинала в S3 staging и транзакции metadata + job + outbox. Статус доступен через GET; content до готовности возвращает 409. +- Добавлены отдельные Valkey Streams broker с AOF и volume, outbox publisher и worker. Доставка at least once, ACK после устойчивого результата; дубли не создают второй набор объектов. +- Worker сохраняет неизменённый оригинал, JPEG thumbnail, центрированный crop 128×128 и optimized WebP по контракту, с учётом EXIF orientation и прозрачности. +- Реализованы пять автоматических попыток, backoff с jitter, lease/recovery, безопасные коды ошибок, реестр failed и явный replay с audit. Ready-задания повторно не запускаются. +- Добавлены ограничения bytes/pixels/времени/памяти, ограниченная конкуренция и очистка staging после фиксации результата либо истечения retention failed. + +### Запуск из чистого clone + +Нужны Docker Engine с Compose v2, Python 3.11, Bash и make. Команды из корня репозитория: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +make check validate PYTHON=.venv/bin/python +./scripts/start.sh +./scripts/check.sh + +# Полная проверка на отдельном временном Docker Compose project: +./scripts/check-docker.sh + +# Остановка обычного стенда с сохранением volumes: +./scripts/stop.sh +``` + +`start.sh` создаёт локальные `.env` и secrets, собирает приложение, выполняет миграции и bootstrap, ждёт готовности обеих API-реплик. API доступен на `http://localhost:8080`. + +### Результаты проверок + +Проверено 12.11.2026: `make check validate test`, 39 собственных API-тестов и все 17 групп полного сценария прошли. Воспроизведены остановка worker и рестарты API/broker, оба разрыва outbox, crash worker после PUT и перед ACK, дубли, исчерпание retry/replay, постоянная ошибка, timeout/MemoryError, очистка staging, недоступность broker, persistence и failover API. Отдельно подтверждён фактический restart всех 11 постоянных контейнеров с сохранением их ID и volumes. + +- [Отчёт, архитектура, команды и ограничения](../REPORT.md). +- [Полный вывод проверок](../evidence/lab5-checks.txt) и [JSON результатов](../evidence/lab5-checks.json). +- [Доказательство рестартов](../evidence/lab5-restart.json) и [вывод persistence](../evidence/lab5-restart.txt). + +### Ограничения / известные ошибки + +Стенд учебный, на одном Docker-host; потеря host/disk/volumes и ручное удаление объектов не покрываются. Поддерживается pipeline_version=1; исследовательские треки на 5 не реализованы. При длительном простое worker outbox и staging требуют свободного диска; общей квоты принятой работы нет. Failed staging хранится 24 часа, после удаления replay запрещён. Превышение ресурсов обработки заканчивается failed после ограниченного числа попыток. Старые ready-вложения lab4 остаются доступны без автоматического создания вариантов. + +### Источники и использование ИИ + +Основа — ТЗ, OpenAPI, `attachment-states.md` и `jobs.schema.json` этого репозитория; официальная документация Redis Streams/XAUTOCLAIM, Valkey persistence, Pillow ImageOps и Python resource. Ссылки приведены в [REPORT.md](../REPORT.md#первичные-источники). + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59c9f4e..ac7f32b 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,55 @@ -## Лабораторная 6 +## Лабораторная 6 — наблюдаемость, оценка 5, трек 2 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Савельев Иван / БВТ2304. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +### Результат -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Сохранён работающий pipeline lab5. Добавлены явный OTel SDK для API/worker/publisher, +Collector, Prometheus, Loki, Tempo и Grafana с provisioning трёх источников и +импортируемым dashboard на 19 панелей. W3C context хранится в durable job/outbox, +проходит через broker и сохраняет цепочку при retry до S3/DB ready. + +Реализованы SLO доступности API 99%/1h и готовности изображения 95% за 30s/1h. +SLI изображений учитывает failed, late и не завершившиеся jobs. Fast/slow burn +alerts используют пары 2m/30s и 6m/1m, guards и for; есть high error rate и queue lag. +Исследование сравнивает короткий всплеск с длительными отказами DB/worker, +измеряет detection/resolution и ложные срабатывания. Разбор причин использует +сохранённые метрики, логи и реальные traces; запросы собраны в runbook. + +### Запуск и проверки + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +./scripts/start.sh +./scripts/check-lab6.sh +``` + +- API: http://localhost:8080. +- Dashboard: http://localhost:3000/d/messenger-lab6, локальный Viewer. +- [Отчёт, результаты, версии и ограничения](../REPORT.md). +- [Runbook](../docs/runbook.md), [методика](../research/README.md). +- API/39 собственных tests: `evidence/lab6-api-checks.txt`. +- 17 групп сохранённых критериев lab5: `evidence/lab6-lab5-regression.json`. +- Trace с retry и логи: `evidence/lab6-trace-retry.json`, `evidence/lab6-correlated-logs.json`. +- Loki outage/ограниченная RAM/измеренные потери: `evidence/lab6-checks.json`. +- Реальные restart/volumes: `evidence/lab6-restart.json`, `evidence/lab6-telemetry-persistence.json`. +- Трек 2: `evidence/lab6-slo-research.json`, `evidence/lab6-slo-series.json`. +- Проверки правил: `evidence/lab6-promtool.txt`. + +### Ограничения + +Один host; потеря диска/volumes не покрыта. Telemetry buffer ограничен и допускает +измеряемые потери при недоступном backend. SLO периода час задан, но один короткий +live эксперимент проверяет детектор, а не долгосрочную надёжность. Отказы LB/сети +до API требуют отдельного внешнего SLI. Внешняя доставка alerts не требуется. + +Источники — ТЗ и официальная документация OTel, W3C, Grafana/Prometheus и Google SRE +Workbook; ссылки в REPORT.md. Реализация и проверки подготовлены с помощью Codex. + +- [x] Минимум предыдущей работы сохранён. +- [x] Обязательные контракты и публичные tests не изменены. +- [x] Три сигнала поступают в реальные persistent stores; dashboard provisioned из Git. +- [x] Секреты, тела сообщений/изображений и SQL/params не экспортируются. +- [x] Локальные .env/secrets и runtime artifacts исключены из Git. +- [x] Условия alerts, окна, for и runbook сохранены в Git. diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 0000000..1f5a529 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,24 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read + +jobs: + docker: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: python -m venv .venv + - run: .venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt + - run: ./scripts/check-lab6.sh + - uses: actions/upload-artifact@v4 + if: always() + with: + name: lab6-evidence + path: | + evidence/lab6-*.txt + evidence/lab6-*.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..64af548 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +COPY requirements.txt . +RUN python -m pip install --no-cache-dir -r requirements.txt \ + && python -m pip check + +RUN groupadd --gid 10001 app \ + && useradd --uid 10001 --gid app --no-create-home app + +COPY src/ ./src/ +COPY migrations/ ./migrations/ +COPY research/ ./research/ + +USER 10001:10001 +EXPOSE 8080 +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=2s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "-m", "src.healthcheck"] + +CMD ["python", "-m", "src"] diff --git a/REPORT.lab5.md b/REPORT.lab5.md new file mode 100644 index 0000000..54334b3 --- /dev/null +++ b/REPORT.lab5.md @@ -0,0 +1,184 @@ +# Отчёт — лабораторная 5 + +- Автор, группа: Савельев Иван, БВТ2304. +- Целевая оценка: **4** — все требования уровней 3 и 4. Исследовательский трек оценки 5 не заявлен. +- Стенд: Docker Compose; HAProxy, две реплики FastAPI, PostgreSQL, Valkey для сессий, SeaweedFS S3; отдельные Valkey Streams broker, outbox publisher и worker. +- Commit/PR: после публикации. Evidence содержит source_commit и working_tree=true: проверялось рабочее дерево поверх указанного commit. +- Использование ИИ: перенос базы, код, проверки и отчёт подготовлены с помощью Codex. Результаты проверок получены реальным запуском Docker, а не предполагаются по коду. + +## Запуск из чистого clone + +Нужны Docker Engine с Compose v2, Python 3.11, Bash и make. Все команды выполняются из корня репозитория. Внешние заранее установленные БД/брокер/S3 не нужны. + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +make check validate PYTHON=.venv/bin/python +./scripts/start.sh +./scripts/check.sh +``` + +`start.sh` создаёт локальные `.env` и `secrets/`, собирает образ, выполняет миграции и bootstrap приватного bucket, поднимает сервисы и ждёт обе API-реплики через LB. Существующие настройки и пароли сохраняются. `.env`, secrets, виртуальное окружение и runtime data исключены из Git; Docker context содержит только исходники, зависимости и миграции. + +API: http://localhost:8080. Если порт занят, задайте `HOST_PORT=8081` в `.env`; HTTP-проверки запускайте с `BASE_URL=http://localhost:8081`. Compose project `mtusi-lab5`, образы `mtusi-messenger:lab5` и volumes отделены от предыдущих лабораторных. Локальные credentials генерируются заново, файлы из lab4 не копируются. + +```sh +# Полный автоматический сценарий на отдельном временном project и свободном порту: +./scripts/check-docker.sh + +# Обычный стенд, остановка без потери данных: +./scripts/stop.sh +./scripts/start.sh + +# Перезапуск всех постоянных процессов и хранилищ с сохранением volumes: +./scripts/restart.sh + +# Только для намеренного удаления учебных данных ТЕКУЩЕГО Compose project: +docker compose down --volumes +``` + +Полный сценарий создаёт синтетические данные в отдельном `lab5-check-` и удаляет только его контейнеры/volumes. Пишет `evidence/lab5-checks.txt` и `evidence/lab5-checks.json`. Для диагностики сбоя можно явно задать `KEEP_TEST_STACK=1`; это оставит только неуспешный временный стенд. Основной стенд не затрагивается. + +## Что перенесено и изменено + +Из lab4 сохранены регистрация/Argon2id, Basic-вход и Bearer-сессии в Valkey, ACL чатов и вложений, сообщения и keyset-пагинация, X-Request-Id/X-Instance-Id, общая идемпотентность сообщений, две API-реплики за LB, приватный S3, лимиты загрузки, миграции и эксплуатационные скрипты. Предыдущие отчёты — `REPORT.lab2.md`, `REPORT.lab3.md`, `REPORT.lab4.md`; старые evidence относятся к соответствующим работам. + +Объявленный переход API: upload возвращает **202 со снимком queued**, а клиент опрашивает GET metadata. Queued/processing уже можно прикрепить к сообщению в том же чате. Для failed новое прикрепление даёт 409. Content любого не-ready вложения даёт 409, отсутствующий вариант ready — 404. Оригинал скачивается побайтно без перекодирования и redirect; API не раскрывает storage keys/credentials. Контракт, README задания и публичные tests не изменялись. + +Миграции 001–003 сохранены. Миграция 004 добавляет состояния/metadata вариантов и таблицы jobs, outbox, audit событий. Старые ready-вложения сохраняют свои ключи и читаются после обновления; варианты у старых записей не создаются автоматически. Схема описана в `docs/schema.md`. + +## Архитектура и границы устойчивости + +```text +Клиент → HAProxy → API ×2 → PostgreSQL: metadata + job + outbox + → приватный S3 staging + +PostgreSQL outbox → publisher → Valkey Streams (AOF, volume) + ↓ XREADGROUP, prefetch=1 + worker (2 процесса) + ↓ + отдельный resize subprocess + ↓ + S3 original + thumbnail + crop + optimized + ↓ + PostgreSQL COMMIT ready → broker ACK + +cleanup → orphan PUT journal + staging только ready/просроченных failed +``` + +PostgreSQL — источник истины для намерения обработки и состояния. Valkey Streams обеспечивает at-least-once доставку. Это отдельный service `broker`, не Pub/Sub и не тот же экземпляр, что сессии. Stream `attachments:v1`, группа `image-workers`, начало чтения `0`; topology создаётся идемпотентным XGROUP CREATE MKSTREAM. Сообщения не trim-ятся до ACK. После XACK запись удаляется XDEL в одной broker transaction; используется одна группа. + +1. API принимает все bytes с ограничением размера, проверяет имя, MIME и настоящий PNG/JPEG. Resize/генерация вариантов в API не выполняются. +2. Перед PUT фиксируется журнал `attachment_uploads`. Полный проверенный оригинал записывается в S3 `attachments/staging/`. Это тот же приватный S3, но отдельный staging prefix; на этом этапе конечного original/вариантов ещё нет. +3. Одна транзакция PostgreSQL вставляет queued metadata, job и outbox и удаляет запись журнала. **202 отправляется после PUT и успешного COMMIT.** Локальный TemporaryFile нужен только во время HTTP-приёма, устойчивым staging не является. +4. Отдельный publisher читает due outbox с блокировкой SKIP LOCKED и делает XADD. Broker настроен AOF `appendfsync always`, volume, `noeviction`. Успешный XADD подтверждает приём брокером с этой настройкой persistence; это не consumer ACK. После ответа dispatcher отмечает published_at. +5. Worker захватывает job транзакционно: status=processing, случайный lease_token и deadline. Данные для скачивания берёт из БД, а не из сообщения/filename. Проверяет размер и SHA-256 staging. Записывает original и все три варианта, затем в одной транзакции ставит ready и metadata/checksums. Только после COMMIT подтверждает доставку. + +В сообщении только поля `jobs.schema.json`: schema_version, job_id, type, attachment_id, pipeline_version=1, attempt, created_at. Bytes, URL, credentials, filename и storage key отсутствуют. job_id постоянен при redelivery и managed retry; attempt начинается с 1 и меняется при новой управляемой попытке. Native stream ID — отдельный идентификатор доставки. + +## Идемпотентность, retry и replay + +Ключ эффекта `(attachment_id, pipeline_version, variant)` реализован детерминированными объектами: + +```text +attachments//v1/original +attachments//v1/thumbnail +attachments//v1/crop +attachments//v1/optimized +``` + +Число объектов конечного результата — четыре. Дубль queued/processing не захватывает уже арендованную job. Устаревшая попытка не меняет актуальное состояние. Перед PUT и COMMIT worker блокирует строки job/attachment и сверяет lease_token и deadline. При повторе после незавершённого PUT те же детерминированные объекты могут записаться ещё раз, второй набор не создаётся. Дубль ready проверяет полную metadata и размеры объектов через S3 HEAD, затем ACK, не меняя status/updated_at и не выполняя resize. В этой работе поддерживается только pipeline_version=1; трек versioned pipeline на 5 не заявлен. + +Временные ошибки S3, resource limit и timeout дают queued с новым due outbox. Backoff: `uniform(0.5,1.5) × min(60, JOB_BACKOFF_SECONDS × 2^(attempt-1))`; интервал сохраняется в PostgreSQL. Default base=2 s, jitter присутствует. Автоматический бюджет — `JOB_MAX_ATTEMPTS=5`. При исчерпании — failed/error_code=retry_exhausted; последняя безопасная причина сохраняется в job. Нарушение checksum/размера staging или невалидное изображение — постоянный invalid_image, сразу failed без повторов. Тексты исключений и stack traces не передаются в API/registry. + +Publisher также восстанавливает истёкший processing lease: записывает worker_lost и создаёт следующую попытку либо failed при исчерпании бюджета. Это не бесконечный retry. Pending-доставки умершего consumer забираются XAUTOCLAIM после lease timeout. Если recovery уже создал новую попытку, старое сообщение становится obsolete и получает ACK; намерение следующей попытки уже устойчиво сохранено. + +Реестр failed и явный replay: + +```sh +docker compose exec -T worker python -m src.job_admin failed +docker compose exec -T worker python -m src.job_admin replay +``` + +Replay разрешён только для failed с ещё сохранённым staging. Он сохраняет job_id/ключ эффекта, записывает operator_replay в audit, увеличивает глобальный attempt и выделяет новый ограниченный бюджет из пяти попыток. Ready/queued/processing повторять этой командой нельзя; повтор ready завершается ошибкой без изменения БД/объектов. Глобальный attempt монотонен, поэтому старые сообщения не запускают replay как новую работу. После удаления failed staging replay запрещён: исходник надо загрузить заново. HTTP endpoint управления заданиями не добавлен. + +## Правила изображений и ограничения ресурсов + +- Оригинал остаётся исходными bytes, MIME, size_bytes и SHA-256 относятся к нему. +- EXIF orientation применяется к производным изображениям. Прозрачность компонуется на белый; производные RGB, без EXIF/ICC/comment metadata. +- Thumbnail: JPEG quality=85, вписывание в 256×256 без увеличения, целочисленное округление вниз с минимумом 1 px. Fixture 320×200 → 256×160. +- Crop: JPEG quality=85, центрированный квадрат 128×128, LANCZOS; маленький исходник может увеличиваться. +- Optimized: WebP quality=80, method=4; вписывание в 1280×1280 без увеличения. Уменьшение байтов каждого маленького файла не обещается. +- Загрузка ≤10 MiB и ≤40 млн pixels. Limit проверяется до полной распаковки; Content-Length и фактический поток проверяются независимо. Corrupt/MIME spoof — 415, bytes/pixels overflow — 413. API допускает одну проверяемую загрузку на реплику, таймаут тела 30 s и ожидания слота 20 s. +- Worker concurrency=2, каждый consumer получает максимум одну доставку за раз. Для заданного Compose memory budget разрешено concurrency 1 или 2. +- Каждый resize выполняется отдельным subprocess: wall timeout 60 s, CPU limit 60/61 s, address-space limit 640 MiB (`RLIMIT_AS`). При timeout subprocess убивается; MemoryError/resource termination не убивает основной consumer и проходит bounded retry. Worker container ограничен 1536 MiB, 2 CPU и 96 PID. Lease по умолчанию 90 s и должен превышать resize timeout минимум на 5 s. +- S3 операции имеют connect timeout 1 s/read timeout 3 s, SDK retries отключены. PostgreSQL I/O, pool wait и блокировки ограничены. CPU/resize выполняется вне DB-транзакции; транзакция публикации результата включает только fencing, четыре ограниченных PUT и COMMIT. + +`WORKER_TASK_MEMORY_MB` ограничивает адресное пространство процесса, а Compose mem_limit — весь контейнер. Это разные ограничения; поддержка 40 млн pixels не обещает, что любой формат/ориентация уложится в память: превышение ресурсов заканчивается наблюдаемым failed после ограниченного бюджета, а не бесконечной обработкой. + +## Staging retention и очистка + +`cleanup` запускается раз в 30 s. Журнал orphan PUT старше `UPLOAD_CLEANUP_GRACE_SECONDS=3600` очищается под advisory lock: свежая/активная загрузка не удаляется. Crash после staging, но до metadata/job не дал 202; его журнал позволяет удалить orphan без обхода всего bucket. + +Staging queued/processing не удаляется независимо от возраста. После безопасного COMMIT ready staging удаляется при ближайшем проходе; конечные четыре объекта сохраняются. Failed staging и частичные конечные объекты хранятся `STAGING_RETENTION_SECONDS=86400` после последнего перехода в failed, затем удаляются; metadata, failed registry и audit остаются. Операции DELETE идемпотентны, отметка staging_deleted_at фиксируется после удаления. Cleaner и replay блокируют те же строки, поэтому не удаляют staging текущей replay-работы. + +## Сервисы, настройки и bootstrap + +| Роль / service | Версия | Readiness / persistence / credentials | +| --- | --- | --- | +| API / api ×2 | Python 3.11.15; FastAPI 0.142.4 | non-root UID 10001, health live/ready; 384 MiB/1 CPU на реплику; Docker secrets | +| БД / postgres | PostgreSQL 17.6-bookworm | pg_isready, postgres_data; отдельный migrate с checksum/advisory lock | +| Сессии / valkey | Valkey 8.1.3 | PING, AOF always, valkey_data, внутренний порт | +| Брокер / broker | Valkey 8.1.3 | авторизованный PING, AOF always, broker_data, отдельный пароль через secret | +| Outbox / publisher | тот же собранный Python image | restart unless-stopped; DB+broker secrets; topology bootstrap идемпотентен | +| Worker / worker | тот же Python image; Pillow 13.0.0 | restart supervisor, 2 consumer-процесса, leases; DB+broker+S3 secrets | +| S3 / s3 + s3-storage | SeaweedFS 3.85 с digest | gateway probe + filer HTTP; s3_data; storage только в s3-internal | +| S3 bootstrap / s3-bootstrap | MinIO SDK 7.2.20 | root secrets только для создания приватного bucket; повтор безопасен | +| LB / lb | HAProxy 3.2.6-alpine | readiness backend; единственный порт 127.0.0.1:8080 | +| Cleanup / cleanup | тот же Python image | restart unless-stopped, DB+S3 secrets; без broker credentials | + +Python base image закреплён digest в Dockerfile; S3 image — digest в Compose. Все Python-зависимости закреплены в requirements.txt. Административные интерфейсы S3 и внутренние API/БД/broker ports наружу не опубликованы. S3 application credentials разрешают работу только своего bucket/prefix attachments/*; root используется только bootstrap. API/cleaner не получают broker password. `.env.example` содержит пути secret-файлов, а не их значения. + +Соответствие настроек: APP_HOST/APP_PORT — адрес API внутри контейнера; HOST_PORT — порт LB; APP_ENV — учебная метка. SESSION_STORE=valkey и SESSION_TTL_SECONDS=3600 — общее session storage; CACHE_URL_FILE/DATABASE_URL_FILE — Docker secrets. API_REPLICAS=2; IDEMPOTENCY_TTL_SECONDS≥86400. S3_ENDPOINT/REGION/BUCKET и S3_*_FILE — приватный bucket. MAX_UPLOAD_BYTES/MAX_IMAGE_PIXELS и UPLOAD_CLEANUP_GRACE_SECONDS — приём и orphan cleanup. BROKER_URL_FILE — авторизованный Redis-compatible URL отдельного broker. WORKER_CONCURRENCY, WORKER_TASK_MEMORY_MB, JOB_TIMEOUT_SECONDS, JOB_LEASE_SECONDS, JOB_MAX_ATTEMPTS, JOB_BACKOFF_SECONDS — ограничения/повторы. STAGING_RETENTION_SECONDS — срок хранения failed исходников. + +/health/ready API зависит от PostgreSQL, сессий и S3: они нужны синхронному пути. Broker/worker/publisher не являются зависимостями readiness API: при их остановке original+outbox надёжно принимаются, текстовые сообщения работают, resize синхронно не запускается. У фоновых процессов нет HTTP health endpoint; состояние jobs/pending/failed наблюдается через БД, broker и операторскую CLI. + +## Доказательства критериев + +Проверено 12.11.2026: все **17 групп проверок** и **39 API regression tests** прошли. Docker 29.0.1, aarch64, 10 CPU, 8 217 448 448 bytes RAM Docker VM. + +Полный воспроизводимый запуск: `./scripts/check-docker.sh`. Исходник сценария — `student-tests/lab5_pipeline.py`. Результаты и реальные job/attachment IDs — `evidence/lab5-checks.json`, консольный вывод — `evidence/lab5-checks.txt`. Дополнительное доказательство реального restart обычного стенда с прежними container IDs и volumes — `evidence/lab5-restart.json`; воспроизводится `.venv/bin/python student-tests/verify_restart.py`. + +| Критерий README | Команда / ключ проверки evidence | Наблюдаемый результат | +| --- | --- | --- | +| Сохранён lab4, переход 201→202, smoke | make check validate test; public_smoke_and_schema, student_api_regressions | Публичный lab5 сценарий и собственные API regression tests | +| Отдельные broker/worker, persistence, лимиты | topology_and_limits | Реальные контейнеры/volumes, две API, broker AOF always, worker concurrency=2/prefetch=1 | +| Worker остановлен → 202 → сообщение → рестарты → ready | worker_offline_api_broker_restart | Тот же attachment/job после рестарта API и broker; 409 до ready; original/три варианта доступны | +| Original/thumbnail/crop/WebP по контракту | image_semantics | Bytes/checksums/MIME и реальные размеры; small/alpha/EXIF/floor cases | +| Outbox, crash COMMIT→publish | outbox_crash_before_publish | Устойчивое намерение доставляется после автоматического restart publisher | +| Неопределённый XADD, дубли | outbox_crash_after_publish | Повтор publish не создаёт второго результата | +| Crash после side effect до COMMIT/ACK | worker_crash_after_objects, worker_crash_before_ack | Lease/reclaim восстанавливают работу; четыре canonical объекта; ready дубль — no-op | +| Retry/backoff/jitter, failed registry, replay | bounded_retry_failed_registry_replay | 5 автоматических неудач → failed; audit содержит время/попытки; явный replay → ready; ready replay отклонён | +| Постоянная ошибка | permanent_error | invalid_image с первой попытки, безопасный код | +| Ограничение времени и памяти обработки | image_subprocess_timeout, image_subprocess_memory | Реальное убийство зависшего subprocess и MemoryError при выделении 1 GiB под лимитом 640 MiB; следующий retry завершает тот же job | +| Безопасная очистка staging | staging_cleanup | Active staging сохранён; ready/expired failed удалены; replay без исходника запрещён | +| Broker недоступен | broker_offline_outbox | API принимает 202 и текст; outbox доставляет после восстановления | +| Сохранность после всех рестартов | make test-persistence; public_persistence | Сессия, история и original переживают рестарты с volumes | +| Отказ API | make test-failover; public_api_failover | Другая реплика продолжает обслуживать подтверждённые сообщения без дублей | + +Failpoints включаются только явным `ENABLE_FAULT_INJECTION=1` во временном тестовом project; по умолчанию отключены. Файлы управления доступны только через docker exec внутри контейнера, HTTP управления ими нет. `before_publish` завершает publisher после DB COMMIT намерения до XADD; `after_publish` — после XADD до фиксации outbox. `after_objects` завершает consumer после PUT четырёх объектов до COMMIT ready; `before_ack` — после COMMIT ready до XACK. Маркеры срабатывания проверяются тестом, поэтому простой быстрый успех без внедрённого сбоя не засчитывается. Отдельный timeout failpoint усыпляет именно ограниченный resize subprocess, не подменяет результат mock-объектом. + +## Эксперименты и ограничения + +Это проверка критериев на 4; исследование максимального throughput, сравнение concurrency/prefetch и versioned pipeline не выполнялись. Полный сценарий использует concurrency=2/prefetch=1, max_attempts=5; для быстрых fault checks timeout=5 s, lease=15 s, base backoff=0.15 s. Production-подобные default настройки остаются 60/90/2 s в .env.example. Дата, Docker/hardware и лимиты конкретного запуска записаны в evidence. Измерения latency/RSS этой проверки не трактуются как нагрузочный benchmark. + +Обработка at-least-once, не exactly-once: XADD и PUT могут повториться, идемпотентен конечный эффект и его публикация. Сбой host/disk с потерей volumes, потеря единственной PostgreSQL/S3 или ручное удаление объектов не покрываются. Backup/replication этих хранилищ в lab5 не добавлялись. Очередь ограничена maxmemory/noeviction broker; outbox и staging во время длительного простоя требуют диска, общей квоты принятой работы эта оценка не требует. Повторы самого HTTP upload могут создать отдельные вложения по контракту. + +## Первичные источники + +- [Задание и объявленные переходы](README.md), [состояния](contracts/attachment-states.md), [схема job](contracts/jobs.schema.json). +- [Redis Streams: consumer groups, pending, acknowledgements](https://redis.io/docs/latest/develop/data-types/streams/). +- [XAUTOCLAIM: восстановление pending](https://redis.io/docs/latest/commands/xautoclaim/). +- [Valkey persistence: AOF и fsync](https://valkey.io/topics/persistence/). +- [Pillow ImageOps: EXIF transpose и центрированный fit](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html). +- [Python resource limits](https://docs.python.org/3/library/resource.html). diff --git a/REPORT.md b/REPORT.md index b97ae42..92e9162 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,221 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Савельев Иван, БВТ2304. +- Целевая оценка: **5**, трек 2 — **SLO и обнаружение инцидентов**. Реализованы уровни 3 и 4. +- Основа: готовая lab5, включая асинхронный pipeline и все её проверки на 4. Исторический отчёт — [REPORT.lab5.md](REPORT.lab5.md). +- Commit/PR: рабочее дерево поверх исходного skeleton; публикация выполняется отдельно. Проверенные файлы перечислены с SHA-256 в `evidence/lab6-source-manifest.json`. +- Помощь ИИ: Codex использован для переноса, реализации, проверок и подготовки отчёта. Evidence получен реальными запусками Docker и запросами к хранилищам. +- Проверено 19.11.2026 на macOS arm64, Docker Desktop 29.0.1: 10 CPU, 8217448448 bytes RAM (≈7.65 GiB) доступны Docker VM. Это учебный стенд на одном host. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine/Desktop с Compose v2, Python 3.11, Bash и make. Интернет нужен для первой установки зависимостей и загрузки закреплённых образов. Все команды из корня: -## Архитектура и решения +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +make check validate PYTHON=.venv/bin/python +./scripts/start.sh +./scripts/check.sh +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`start.sh` создаёт локальные `.env` и `secrets/`, собирает приложение, применяет миграции и bootstrap приватного S3 bucket, поднимает API/worker и наблюдаемость. Затем ожидает две API-реплики через LB, scrape Collector, все хранилища и provisioning Grafana. Ручной импорт dashboard не нужен. + +- API: http://localhost:8080; health: `/health/live`, `/health/ready`. +- [Grafana dashboard](http://localhost:3000/d/messenger-lab6): локальный anonymous Viewer; пароли приложения в Grafana не вводятся. +- Порты меняются через `HOST_PORT` и `GRAFANA_PORT` в `.env`; для API-тестов передать соответствующий `BASE_URL`. Драйвер наблюдаемости определяет опубликованный порт Grafana через Compose; можно задать `GRAFANA_URL`. +- Compose project `mtusi-lab6`, приложение `mtusi-messenger:lab6`; volumes отделены от lab5. Локальные `.env`/secrets не входят в Git и Docker build context. В новом clone credentials генерируются заново. + +```sh +# Весь минимум lab5 + API/telemetry/persistence + исследование lab6: +./scripts/check-lab6.sh + +# Только исследование трека 2 на уже запущенном синтетическом стенде (~11 минут): +./scripts/research.sh + +# Остановка с сохранением данных / повторный запуск: +./scripts/stop.sh +./scripts/start.sh + +# Перезапуск процессов и хранилищ без удаления volumes: +./scripts/restart.sh + +# Только намеренное удаление данных СВОЕГО учебного Compose project: +docker compose down --volumes +``` + +Полный script сначала проверяет lab5 в отдельном временном project на свободных портах и новых volumes, затем удаляет только этот временный стенд. Исследование lab6 использует текущий локальный синтетический стенд и намеренно создаёт ограниченные отказы. Стандартная работа приложения не требует `ENABLE_FAULT_INJECTION`; проверка retry временно включает его только для worker и затем выключает. + +## Архитектура и версии + +```text +HTTP → HAProxy → API ×2 → PostgreSQL / Valkey sessions / private S3 + └ durable metadata + job + outbox (W3C context) + → publisher → Valkey Streams → worker ×2 + → S3 GET/PUT → DB ready → ACK + +API / publisher / worker → OTLP HTTP → Collector + ├ Prometheus exporter → Prometheus ← scrape + ├ OTLP HTTP → Loki + └ OTLP gRPC → Tempo + ↑ + Grafana: metrics / logs / traces +``` + +API, ACL, сессии, идемпотентность, ограничения изображений, at-least-once/retry/replay и очистка staging перенесены из lab5. Обязательные контракты и публичные tests сохранены. Миграции 001–004 не изменены; 005 добавляет сохранённый trace context и `ready_at`. Новые поля не меняют бизнес-API. Источник истины — PostgreSQL; ACK по-прежнему выполняется после устойчивого результата/retry. + +| Роль / service | Закреплённая версия | Persistence / readiness / credentials | +| --- | --- | --- | +| API / worker / publisher / cleanup / migrate | Python 3.11.15, FastAPI 0.142.4; image `mtusi-messenger:lab6` | Исходники и lock версий в requirements; API `/health/ready`, worker читает broker; отдельный migrate проверяет checksum. Docker secrets, non-root UID 10001 | +| PostgreSQL / postgres | 17.6-bookworm | `postgres_data`, pg_isready; password/DSN из secrets | +| Сессии / valkey | 8.1.3 | `valkey_data`, AOF fsync always; ping; только внутренняя сеть | +| Очередь / broker | Valkey 8.1.3 | `broker_data`, AOF fsync always; authenticated ping; password/URL из secrets | +| S3 / s3, s3-storage | SeaweedFS 3.85 с digest | `s3_data`, внутренний TCP/HTTP readiness; private bucket и service account создаёт bootstrap; secrets | +| LB / lb | HAProxy 3.2.6-alpine | `/health/ready`; только `127.0.0.1:8080`; API-порты напрямую не опубликованы | +| Collector / otel-collector | contrib 0.123.0 | Config `infra/otel/collector.yaml`, health 13133 и self metrics 8888 внутри сети; долговременной БД не является | +| Метрики / prometheus | 3.4.1 | `prometheus_data`, retention 7d, scrape/evaluation 5s; rules в Git | +| Логи / loki | 3.5.1 | `loki_data`, TSDB v13/filesystem, retention 7d, native OTLP со structured metadata | +| Трассы / tempo | 2.8.1 | `tempo_data`, WAL + local blocks, retention 7d; OTLP receiver только внутри сети | +| Grafana / grafana | 12.0.2 | `grafana_data`, `/api/health`; 3 datasources + dashboard provisioned из Git; localhost:3000, Viewer | + +Grafana запрашивает хранилища, сама не хранит три сигнала. Collector принимает/буферизует/экспортирует, долгую историю хранят backends. У всех контейнеров заданы CPU/RAM limits; точные значения, volumes и опубликованные порты — `lab6-runtime.json`. Collector: 1 CPU/256 MiB; Prometheus/Loki/Grafana: по 1 CPU/384 MiB; Tempo: 1 CPU/512 MiB с GOMEMLIMIT 384 MiB. Приложение сохранило ограничения lab5. + +SDK/API/exporters OTel **1.45.1**, semantic conventions **0.66b1**; полный список зависимостей закреплён в `requirements.txt`. Используются явные span/metric имена, а не автосбор SQL/HTTP headers. Встроенная автоматическая telemetry FastAPI отключена: один SDK и один набор инструментов без дублирования сигналов. Counter series для известных маршрутов и status classes инициализированы нулём до первого scrape: короткий первый всплеск ошибок не теряется из rate из-за отсутствия исходного нулевого sample. Фиктивные запросы или histogram observations для этого не создаются. + +`.env.example` соответствует настройкам: `DATABASE_URL_FILE`, `CACHE_URL_FILE`, S3 и `BROKER_URL_FILE` читают Docker secrets; worker concurrency/retry/lease/time/memory — перенесённые настройки pipeline; `OTEL_EXPORTER_OTLP_ENDPOINT` — общий Collector, protocol HTTP/protobuf. Resource `service.name` устанавливается отдельно для API, worker и publisher; `service.version=6.0.0`, instance — hostname:pid, environment — lab. `OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED=true` включает контроль очередей SDK. Перезапуск сохраняет volumes всех signal stores. + +## Сигналы и запросы + +[Dashboard JSON](infra/grafana/dashboards/messenger.json) содержит 19 панелей; datasources/provider рядом в `infra/grafana/provisioning/`. `lab6-dashboard-checks.json` подтверждает выполнение запросов ко всем трём источникам. + +| Реальное имя в Prometheus | Тип / единицы / labels | +| --- | --- | +| `messenger_http_requests_total` | counter запросов; method, route template, status_class; health исключён | +| `messenger_http_duration_seconds_*` | histogram, секунды до response headers; те же labels | +| `messenger_job_attempts_total` | counter попыток; job_type, outcome (ready/retry/failed/error/duplicate/busy/obsolete/invalid) | +| `messenger_job_duration_seconds_*` | histogram длительности consumer attempt, секунды; job_type/outcome | +| `messenger_attachment_ready_duration_seconds_*` | histogram acceptance → ready успешных jobs, секунды; job_type | +| `messenger_queue_depth` / `messenger_queue_oldest_age_seconds` | gauges: незавершённые durable jobs / возраст самой старой, секунды; queue=attachments | +| `messenger_db_pool` | gauge соединений; state=used/waiting | +| `messenger_worker_busy` / `messenger_worker_capacity` | gauges занятых/доступных worker процессов | +| `messenger_process_cpu_time_seconds_total` / `messenger_process_memory_bytes` | process CPU seconds / RSS bytes; группировка по service_name | +| `messenger_ready_deadlines_total` | durable cohort counter; outcome=good/bad, включая failed/stuck/late jobs | + +`_bucket`, `_sum`, `_count` — части одного histogram. Resource labels добавляются Collector; технический instance ID нужен для суммирования процессов и различения перезапусков. User/chat/message/attachment/request IDs, raw URL, filename и текст labels не являются. Лимит экспирации неактивных metric series Collector — 30s. + +```promql +# RPS (без health), ошибки 5xx / все business requests: +sum(rate(messenger_http_requests_total[1m])) +lab6:api_bad_ratio:1m + +# p95 по всем API-репликам; для p50/p99 меняется quantile: +histogram_quantile(0.95, + sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) + +max(messenger_queue_depth) +max(messenger_queue_oldest_age_seconds) +sum by (state, service_name) (messenger_db_pool) +sum(messenger_worker_busy) / sum(messenger_worker_capacity) +``` + +Buckets в секундах: .005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 20, **30**, 60, 120, 300, 600, +Inf. Percentiles считаются из суммарных buckets, не средних latency/p95 реплик. Backlog не подменяет processing duration. `s3.get_object` измеряет получение ответа S3; чтение stream входит в общий consumer span. + +Лог — структурный JSON с timestamp, severity, service.name, event, trace_id/span_id; HTTP содержит также `request_id`, совпадающий с X-Request-Id. В ответе есть X-Trace-Id. Только job/attachment IDs разрешены в явных диагностических событиях для синтетических заданий; в labels их нет. Пароли, Authorization/session tokens, bodies, изображения, filenames/storage keys/signed URLs и SQL/params не собираются. Exception events с текстом/stack trace отключены; сохраняется только класс ошибки. Loki индексирует service/environment, instance — structured metadata. + +W3C traceparent/tracestate сохраняются в job той же транзакцией, что metadata/outbox. Producer восстанавливает parent, инжектирует свой context в broker payload. Consumer извлекает его; retry сохраняет актуальный context для следующей публикации. После перезапуска процессов связь сохраняется. [Сохранённый trace с retry](evidence/lab6-trace-retry.json) и [логи](evidence/lab6-correlated-logs.json) показывают реальную цепочку API → producer → consumer → retry → producer → consumer → S3/DB. Sampling 100% для учебного стенда. + +## SLI/SLO и alerts + +- **API availability ≥99% за скользящий час**. Bad — instrumented business responses 5xx; denominator — все business requests, включая 4xx; health исключён. Неверный пароль не является серверной ошибкой. Нужны внешние probes для отказов LB/сети до API; в этот SLI они не входят. +- **Ready ≥95% за ≤30s, окно час**. Acceptance timestamp — `job.created_at` в durable metadata transaction; запись становится видна после COMMIT. По `ready_at` оценивается каждый cohort после истечения deadline. Позднее завершение, failed и незавершившееся задание считаются bad. Часовое окно относится к созревшим cohorts, acceptance сдвинута на 30s. В отличие от histogram только successful jobs, SLI не скрывает остановку worker. + +Обоснование: 99% availability задаёт заметный бюджет серверных ошибок для учебного single-host стенда с единственной БД; это проверяемая цель, а не обещание production SLA. Для асинхронного изображения 30s оставляют запас на S3, ожидание публикации и краткий retry; минуты без обработки уже нарушают пользовательское ожидание. Цель 95% допускает редкие тяжёлые изображения/повторы, сохраняя чувствительность к остановке worker. При нашем постоянном профиле час содержит около 9000 HTTP requests и 1800 jobs: budgets — 90 ошибок и 90 просроченных jobs. Четыре ошибки короткого всплеска расходуют около 4.4% часового API budget, ниже порогов эскалации 20%/30%. Час выбран для воспроизводимого учебного исследования; production потребует отдельного обоснования периода и целей. + +| Alert policy | Long / short window | Burn rate / порог API / ready | for / guards | +| --- | --- | --- | --- | +| Fast | 2m AND 30s | 6× / 6% / 30% bad | 15s; long ≥30 API / ≥5 jobs, short ≥2 события | +| Slow | 6m AND 1m | 3× / 3% / 15% bad | 30s; те же guards | +| APIHighErrorRate (сравнение) | 30s | >5% 5xx | 5s | +| AttachmentQueueLag | текущие age/depth | oldest >20s и depth >0 | 10s | + +Fast и slow одновременно проверяют оба окна. Это учебный **1h SLO**: fast соответствует расходу 20% бюджета за 2m, slow — 30% за 6m (`burn × long_window / 1h`). Production окна из примера 30-дневного SLO не копируются механически. Guards снижают шум при малом числе событий. При нулевом denominator ratio — NaN, а не искусственные 0% ошибок/100% availability; отдельно проверяются scrape и трафик. Есть CollectorUnavailable warning. + +Rules/условия/описания/for/runbook находятся в [alerts.yaml](infra/otel/alerts.yaml). Доставка наружу не нужна; firing/inactive (resolved) наблюдаются через Prometheus API и панель ALERTS. [Runbook](docs/runbook.md) содержит запросы и порядок диагностики другим студентом. + +## Эксперимент трека 2 и разбор инцидентов + +Гипотеза, заранее заданная методика, нагрузка и ограничения — [research/README.md](research/README.md). Один real-time прогон: 2 сообщения/s + 0.5 PNG/s, 16 max in-flight, без retries; baseline 120s, spike БД 4s, recovery 60s, lock messages 90s, recovery 100s, worker stopped 120s, recovery 120s. SQL-lock ограничен pg_sleep; worker восстанавливается в finally. Каждые 5s читаются сохранённые alert states/метрики. Исследование выполняется отдельно от полного lab5 regression, чтобы они не конкурировали за ресурсы. + +Финальный прогон **19.11.2026, 18:26:14–18:36:34 МСК**, 619.4s. 1544 ответов, пропущенных arrivals: 0. p50/p95/p99 ниже рассчитаны по всем ответам, включая 5xx; successful RPS — только 2xx. +[Открыть этот интервал в Grafana](http://localhost:3000/d/messenger-lab6?from=1795101974987&to=1795102594377). + +| Фаза | Запросы / 5xx | Successful RPS | p50 / p95 / p99, ms | +| --- | --- | --- | --- | +| Baseline | 301 / 0 | 2.51 | 16.6 / 31.3 / 35.2 | +| Короткий DB spike, 4s | 10 / 4 | 1.50 | 1670.7 / 2017.2 / 2017.2 | +| После короткого всплеска | 150 / 0 | 2.50 | 16.5 / 30.7 / 32.8 | +| Длительное ожидание DB, 90s | 225 / 176 | 0.54 | 2011.8 / 2022.5 / 2025.8 | +| После восстановления DB | 250 / 0 | 2.50 | 16.3 / 31.0 / 33.4 | +| Worker stopped, 120s | 300 / 0 | 2.50 | 16.2 / 31.3 / 35.5 | +| После восстановления worker | 300 / 0 | 2.50 | 13.1 / 30.7 / 35.3 | + +| Alert | Detection после воздействия, s | Resolved после восстановления, s | +| --- | --- | --- | +| APIAvailabilityFastBurn | 40.2 | 35.1 | +| APIAvailabilitySlowBurn | 60.3 | 65.3 | +| APIHighErrorRate | 25.1 | 35.1 | +| AttachmentQueueLag | 45.2 | 30.1 | +| AttachmentReadyFastBurn | 95.5 | 45.2 | +| AttachmentReadySlowBurn | 131.6 | 75.3 | + +Baseline: **0** firing alerts. Короткий реальный всплеск виден в сохранённой 30s error ratio (peak **6.37%**); burn alerts на него сработали **0** раз. Single-window comparator: **APIHighErrorRate**. +В исследовании false positive означает эскалацию короткого всплеска, заранее принятого допустимым для выбранной SLO policy; предупреждение APIHighErrorRate при этом корректно соответствует собственному порогу. +Все заявленные alerts наблюдались firing и затем inactive/resolved. Сохранённый worker trace подтверждает consumer спустя **94.3s** после upload, уже после восстановления worker. + +| Процесс | Max 1m CPU rate, cores | Max RSS, MiB | +| --- | --- | --- | +| messenger-api | 0.149 | 603.75 | +| messenger-publisher | 0.012 | 147.05 | +| messenger-worker | 0.013 | 319.40 | + +Это maxima точек с шагом 5s; CPU усреднён по минуте, RSS суммирован по процессам сервиса. Два consumer процесса worker показаны суммарно. Исходные samples, timestamps и все запросы без credentials сохранены в `lab6-slo-research.json`, ряды — в `lab6-slo-series.json`. + +Причина деградации API локализуется по сохранённым наблюдениям: 5xx и HTTP latency растут, внутри server trace `postgresql.SELECT` занимает около 2s и заканчивается `_WaitTimeout`, тогда как acceptance и worker продолжаются. Это I/O ожидание PostgreSQL, а не искусственный sleep HTTP handler. Конкретные spans/durations — `lab6-db-incident-trace.json`, коррелированные request logs — `lab6-db-incident-logs.json`. Методика внедряет SQL lock, но по одному сохранённому timeout нельзя доказать именно блокировку вместо другой причины медленного ответа БД; этот предел диагностики явно учитывается. + +При остановке worker acceptance/producer продолжаются, successful processing прекращается, capacity исчезает после истечения series, backlog/oldest age растут. В trace принятого upload до восстановления отсутствует consumer; после восстановления очередь дренируется, появляются S3/DB ready spans, но просроченные cohorts сохраняют bad исход. Это проверено по timestamps consumer в `lab6-worker-incident-trace.json` и связанным `lab6-worker-incident-logs.json`. Подробные ряды, CPU/RSS, pool и saturation — `lab6-slo-series.json`. + +## Отказ telemetry backend и границы буферизации + +Collector memory limiter: 192 MiB со spike 32 MiB, check 1s; hard container limit 256 MiB. Batch максимум 128 записей; отдельные exporter queues по 64 batch, 2 consumers, timeout 2s, retry максимум 15s. SDK trace/log queues по 256 записей, batch 64, фоновый export; metric export каждые 5s. Internal SDK queue capacity/size и Collector exporter metrics доступны в Prometheus. Бесконечной буферизации нет: при недоступном backend telemetry теряется, бизнес-запрос не ждёт экспорт. + +Контролируемая остановка Loki проверена `student-tests/lab6_observability.py`: отказ около **51.39s**, **24/24 сообщения записаны**, история читается. После восстановления не найдены **6 из 24** соответствующих request logs; это фактически измеренная потеря по известным request IDs. RAM Collector в четырёх samples — **70.46–72.80 MiB** при limit 256 MiB, OOM=false. Отдельный счётчик failed exports вырос на 6; вообще его нельзя считать числом потерянных записей без проверки из-за retries. Raw результаты — `lab6-checks.json`. Grafana/Prometheus/Tempo сохраняли доступность; Loki после запуска снова принимает OTLP. Потери не маскируются debug exporter или stdout. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. - -## Эксперименты и отказы - -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Критерий | Команда/сценарий | Наблюдение / evidence | +| --- | --- | --- | +| Весь minimum lab5 и сохранённые переходы | `.venv/bin/python -u student-tests/lab5_pipeline.py` (тот же stage в check-lab6) | 17 групп PASS, включая outbox/worker crash, retry/replay, processing limits, cleanup, broker outage, persistence/failover; `lab6-lab5-regression.json/.txt` | +| Контракт, публичный smoke, свои API-тесты с telemetry | make check validate test; unittest discover | PASS и 39 tests OK; `lab6-api-checks.txt` | +| Collector + три persistent stores + Grafana | start.sh; readiness/provisioning | 2 API, CPU/RAM limits, localhost порты, volumes; `lab6-runtime.json` | +| Все обязательные metrics и dashboard queries | lab6_observability; check_dashboard | Реальные series и 19 панелей трёх источников; `lab6-checks.json`, `lab6-dashboard-checks.json` | +| API/worker logs + traces, W3C, retry → S3/DB | Deterministic transient worker failpoint | 59 связанных spans; `lab6-trace-retry.json`, `lab6-correlated-logs.json` | +| Отсутствие секретов/body/ID labels | Canary body/filename/password/token + metric label check | PASS; `lab6-checks.json` | +| Telemetry persistence | Restart всех постоянных контейнеров и запрос старого trace | IDs/volumes сохранены; `lab6-restart.json/.txt`, `lab6-telemetry-persistence.json` | +| Ограниченный отказ Loki | Stop/start Loki, 24 записи под отказом, memory samples | API работает, потери logs измерены, OOM=false; `lab6-checks.json/.txt` | +| SLO/rules и пограничные сценарии | promtool test rules | Spike/sustained/recovery/stalled/no traffic PASS; `lab6-promtool.txt` | +| Два инцидента + трек 2 | research.sh, runbook | Реальные firing/resolved, detection/recovery/false positives, stored observations; `lab6-slo-research.json/.txt`, `lab6-slo-series.json` | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Один host, единственные PostgreSQL/S3/telemetry stores: потеря host/disk/volumes не покрыта. Persistent volumes и проверенный restart не означают HA или backup. Срок хранения telemetry 7d; heap/queues ограничены, при длительном отказе часть наблюдений теряется. Нет внешней доставки alerts и внешнего black-box SLI LB/сети: это не требуется выбранным минимумом. Histogram HTTP измеряет время до headers; полный stream download отдельно не профилируется. Queue depth берётся из durable state БД, а не только broker. + +Эксперимент короче часа и выполнен по одному live прогону каждого отказа на небольшой синтетической нагрузке. Он проверяет работу детектора и воспроизводимость диагностики; не доказывает часовую/месячную надёжность, предел RPS или универсальную частоту ложных срабатываний. Точность detection/resolution ограничена шагом 5s плюс SDK export/scrape. Часовые ratio при неполной истории — только доступная часть окна. CPU/RSS относятся к долгоживущим instrumented процессам; resize subprocess ограничен общим Docker limit и отдельные resource series не экспортирует. SQL cohort scan достаточен для учебного объёма; на больших объёмах понадобится incremental aggregation с сохранением состояния. + +## Первичные источники + +- [OpenTelemetry Collector configuration](https://opentelemetry.io/docs/collector/configuration/) и [internal telemetry](https://opentelemetry.io/docs/collector/internal-telemetry/). +- [OTel Python SDK](https://opentelemetry.io/docs/languages/python/) и [W3C Trace Context](https://www.w3.org/TR/trace-context/). +- [Native OTLP в Loki](https://grafana.com/docs/loki/latest/send-data/otel/). +- [Prometheus histograms](https://prometheus.io/docs/practices/histograms/) и [configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/). +- [Google SRE Workbook: multiwindow, multi-burn-rate alerts](https://sre.google/workbook/alerting-on-slos/) — принцип двух окон, адаптированный к явно выбранному учебному периоду. +- [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/); compatibility TraceQL query проверена по source Grafana v12.0.2 и реальному Tempo search API. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..3b1798b --- /dev/null +++ b/compose.yaml @@ -0,0 +1,289 @@ +name: mtusi-lab6 + +x-application: &application + image: mtusi-messenger:lab6 + env_file: .env + environment: &pipeline-environment + OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true" + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + JOB_BACKOFF_SECONDS: ${JOB_BACKOFF_SECONDS:-2} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_TASK_MEMORY_MB: ${WORKER_TASK_MEMORY_MB:-640} + ENABLE_FAULT_INJECTION: ${ENABLE_FAULT_INJECTION:-0} + cpus: 1 + mem_limit: 384m + +x-postgres: &postgres + image: postgres:17.6-bookworm + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + cpus: 1 + mem_limit: 512m + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 3s + retries: 30 + +services: + postgres: + <<: *postgres + volumes: [postgres_data:/var/lib/postgresql/data] + + valkey: + image: valkey/valkey:8.1.3 + command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "always", "--save", "", "--maxmemory", "64mb", "--maxmemory-policy", "noeviction"] + volumes: [valkey_data:/data] + cpus: 0.5 + mem_limit: 128m + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 2s + timeout: 3s + retries: 30 + + broker: + image: valkey/valkey:8.1.3 + command: ["sh", "-c", 'exec valkey-server --appendonly yes --appendfsync always --save "" --maxmemory 128mb --maxmemory-policy noeviction --requirepass "$$(cat /run/secrets/broker_password)"'] + secrets: [broker_password] + volumes: [broker_data:/data] + cpus: 0.5 + mem_limit: 256m + healthcheck: + test: ["CMD-SHELL", 'REDISCLI_AUTH="$$(cat /run/secrets/broker_password)" valkey-cli ping | grep -q PONG'] + interval: 2s + timeout: 3s + retries: 30 + + publisher: + <<: *application + command: ["python", "-m", "src.publisher"] + secrets: [database_url, broker_url] + restart: unless-stopped + healthcheck: + disable: true + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_healthy} + + worker: + <<: *application + command: ["python", "-m", "src.worker"] + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + restart: unless-stopped + mem_limit: 1536m + cpus: 2 + pids_limit: 96 + healthcheck: + disable: true + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_healthy} + s3-bootstrap: {condition: service_completed_successfully} + stop_grace_period: 8s + + api: + <<: *application + build: . + deploy: + replicas: ${API_REPLICAS:-2} + environment: + <<: *pipeline-environment + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + depends_on: + postgres: {condition: service_healthy} + valkey: {condition: service_healthy} + migrate: {condition: service_completed_successfully} + s3: {condition: service_healthy} + s3-bootstrap: {condition: service_completed_successfully} + stop_grace_period: 8s + + migrate: + <<: *application + command: ["python", "-m", "src.migrate"] + secrets: [database_url] + healthcheck: + disable: true + depends_on: + postgres: {condition: service_healthy} + + s3: + image: chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a + command: ["s3", "-filer=s3-storage:8888", "-ip.bind=0.0.0.0", "-port=9000", "-config=/run/secrets/s3_config"] + secrets: [s3_config] + networks: [default, s3-internal] + depends_on: + s3-storage: {condition: service_healthy} + cpus: 0.5 + mem_limit: 256m + healthcheck: + test: ["CMD", "nc", "-z", "127.0.0.1", "9000"] + interval: 2s + timeout: 2s + retries: 30 + + s3-storage: + image: chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a + command: ["server", "-dir=/data", "-ip=s3-storage", "-ip.bind=0.0.0.0", "-filer", "-master.volumeSizeLimitMB=64", "-volume.max=64", "-volume.preStopSeconds=0"] + volumes: [s3_data:/data] + networks: [s3-internal] + cpus: 1 + mem_limit: 512m + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8888/"] + interval: 2s + timeout: 2s + retries: 30 + + s3-bootstrap: + <<: *application + command: ["python", "-m", "src.bootstrap_s3"] + environment: + S3_ROOT_USER_FILE: /run/secrets/s3_root_user + S3_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password + secrets: [s3_root_user, s3_root_password] + cpus: 0.5 + mem_limit: 128m + healthcheck: + disable: true + depends_on: + s3: {condition: service_healthy} + + cleanup: + <<: *application + command: ["python", "-m", "src.cleanup"] + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + restart: unless-stopped + healthcheck: + disable: true + depends_on: + migrate: {condition: service_completed_successfully} + s3-bootstrap: {condition: service_completed_successfully} + + lb: + image: haproxy:3.2.6-alpine + ports: ["127.0.0.1:${HOST_PORT:-8080}:8080"] + volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro] + cpus: 0.5 + mem_limit: 64m + depends_on: + api: {condition: service_healthy} + stop_grace_period: 10s + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"] + interval: 2s + timeout: 2s + retries: 3 + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.123.0 + command: ["--config=/etc/otelcol/config.yaml"] + volumes: [./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro] + depends_on: [loki, tempo] + cpus: 1 + mem_limit: 256m + restart: unless-stopped + + prometheus: + image: prom/prometheus:v3.4.1 + command: ["--config.file=/etc/prometheus/prometheus.yaml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"] + volumes: + - ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro + - prometheus_data:/prometheus + depends_on: [otel-collector] + cpus: 1 + mem_limit: 384m + restart: unless-stopped + + loki: + image: grafana/loki:3.5.1 + command: ["-config.file=/etc/loki/config.yaml"] + volumes: + - ./infra/otel/loki.yaml:/etc/loki/config.yaml:ro + - loki_data:/loki + cpus: 1 + mem_limit: 384m + restart: unless-stopped + + tempo: + image: grafana/tempo:2.8.1 + environment: + GOMEMLIMIT: 384MiB + command: ["-config.file=/etc/tempo/config.yaml"] + volumes: + - ./infra/otel/tempo.yaml:/etc/tempo/config.yaml:ro + - tempo_data:/var/tempo + cpus: 1 + mem_limit: 512m + restart: unless-stopped + + grafana: + image: grafana/grafana:12.0.2 + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_AUTH_BASIC_ENABLED: "false" + GF_AUTH_DISABLE_LOGIN_FORM: "true" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_PLUGINS_PREINSTALL_DISABLED: "true" + GF_PLUGINS_CHECK_FOR_UPDATES: "false" + GF_NEWS_NEWS_FEED_ENABLED: "false" + ports: ["127.0.0.1:${GRAFANA_PORT:-3000}:3000"] + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + depends_on: [prometheus, loki, tempo] + cpus: 1 + mem_limit: 384m + restart: unless-stopped + + postgres-restore: + <<: *postgres + profiles: [research] + volumes: [restore_data:/var/lib/postgresql/data] + +volumes: + prometheus_data: + loki_data: + tempo_data: + grafana_data: + postgres_data: + valkey_data: + restore_data: + s3_data: + broker_data: + +networks: + s3-internal: + internal: true + +secrets: + broker_password: + file: ./secrets/broker_password + broker_url: + file: ./secrets/broker_url + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_root_user: + file: ./secrets/s3_root_user + s3_root_password: + file: ./secrets/s3_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_config: + file: ./secrets/s3_config diff --git a/docs/runbook.md b/docs/runbook.md new file mode 100644 index 0000000..a873573 --- /dev/null +++ b/docs/runbook.md @@ -0,0 +1,91 @@ +# Runbook: лабораторная 6 + +Открыть [dashboard](http://localhost:3000/d/messenger-lab6). Выбрать время инцидента, +а не только последние пять минут. Все шаги диагностики ниже используют сохранённые +Prometheus/Loki/Tempo данные. Команды управления выполняются **после** диагноза. +Внешняя доставка alerts не настроена: состояние проверяется в Prometheus через +Grafana datasource proxy (`research/observations.py`). + +## api + +1. Проверить `API 5xx / all business requests`, RPS и p95/p99. Health и ошибочный + пароль не являются серверными 5xx. Посмотреть обе пары burn rate: 2m/30s и 6m/1m. +2. В Explore → Loki выполнить + `{service_name="messenger-api"} | json | status >= 500`. + По `request_id` найти ответ пользователя; скопировать `trace_id` или нажать TraceID. +3. В Tempo открыть trace. Развернуть server span и `postgresql.transaction`: + медленный `postgresql.SELECT/INSERT` с `error.type=_WaitTimeout` и длительностью + около 2s указывает на срабатывание I/O timeout ожидания PostgreSQL. Если сервер + успел вернуть `LockNotAvailable`, дополнительно подтверждена блокировка. + По одному `_WaitTimeout` нельзя отличить SQL lock от других причин медленной БД. + Это отличается от медленного handler: + при нём server span длинный, а DB spans короткие. `db.pool{state="used"}` + показывает занятые соединения, `waiting` — ожидание свободного соединения. + Наш эксперимент блокирует только messages, поэтому health остаётся успешным. +4. После определения причины дождаться окончания ограниченного failpoint. + Для досрочного восстановления своего учебного эксперимента: + + ```sh + docker compose exec -T postgres psql -U messenger -d messenger -c \ + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6-incident'" + ``` + +5. Проверить успешные записи, короткие DB spans и спад error ratio. + Alert должен перейти firing → inactive (resolved); long-window ratio может + оставаться высоким, но short window уже должен очищаться. Состояния измеряются + раз в 5s; не объявлять recovery по одному успешному запросу. + +## ready + +1. Проверить `Outstanding jobs` и `Oldest unfinished job age`. Это очередь + незавершённых **durable jobs в БД**, включая outbox/retry/processing; + длина Redis Stream сама по себе не отражает всю принятую работу. +2. Сопоставить worker capacity/busy, process CPU/RSS и rate успешных jobs. + Если capacity исчезла, successful jobs остановились, а acceptance продолжается + и backlog/age растут — worker не обрабатывает очередь. После 30s устаревшие + gauge series удаляются Collector; краткое сохранение старого значения допустимо. +3. В Loki: + `{service_name="messenger-api"} | json | event="attachment.accepted"`. + Открыть trace: виден API → `attachments publish`, но в интервале простоя + нет нового `attachments process`. После восстановления в том же trace появятся + consumer → S3 GET/PUT → PostgreSQL UPDATE. Retry — отдельные producer/consumer + spans с возрастающим `job.attempt` и логом `job.retry`. +4. Восстановить worker: `docker compose start worker`. +5. Проверить уменьшение backlog до 0, age до 0, возврат successful jobs и capacity. + Ready SLO остаётся нарушенным для опоздавших заданий: позднее ready не превращает + уже нарушенный 30s deadline в успешный. Дождаться resolved в обеих парах окон. + +## telemetry + +1. Проверить `up{job="collector"}`, `otelcol_exporter_queue_size` и + `otelcol_exporter_send_failed_log_records_total`. Ошибки экспортёра сами по себе + не равны числу окончательно потерянных записей: возможны retries. +2. При живом Collector и ошибках только `otlphttp/loki` проверить доступность + datasource Loki. Если API принимает сообщения и метрики/traces приходят, + это отдельный отказ хранилища логов, а не отказ бизнес-API. +3. Восстановить `docker compose start loki`. Collector ограничен 256 MiB, + memory limiter 192 MiB, очередь каждого экспортёра — 64 batch, максимальный + batch 128 records, retry не дольше 15s. SDK очереди traces/logs — по 256 записей; + сеть работает в фоновых потоках. При исчерпании лимитов telemetry теряется. +4. Проверить возобновление Loki ingestion и опустошение exporter queue. + В `lab6-checks.json` число потерянных request logs считается по известным + request IDs после восстановления, отдельно от счётчиков неудачных экспортов. + +## SLO и интерпретация + +- API: ≥99% business responses за скользящий час без 5xx. 4xx входят в + denominator, но не numerator. Отказы LB/сети до API требуют внешнего probe + и этим SLI не покрываются. +- Изображения: ≥95% accepted jobs готовы за ≤30s; cohort оценивается после + истечения deadline. Failed и не завершившиеся задания входят в bad. + Скользящее окно — час **созревших** cohorts (acceptance сдвинута на 30s). +- Fast burn: 6× бюджета одновременно за 2m и 30s, `for: 15s`. + Slow burn: 3× за 6m и 1m, `for: 30s`. Это учебные окна для 1h SLO, + а не механически перенесённые production окна для 30 дней. +- Guards: ≥30 API requests / ≥5 mature jobs в long window и ≥2 события в short. + Нет данных — нет утверждения об успешном SLO; проверять scrape и traffic. +- `APIHighErrorRate` — single-window comparator (5%/30s, for 5s), + `AttachmentQueueLag` — отдельный эксплуатационный warning (>20s, for 10s). + Сравнивать их с burn alerts, а не считать любой warning доказательством + исчерпания бюджета. Данные одного короткого эксперимента не доказывают + соблюдение SLO в течение полного часа или отсутствие ложных срабатываний вообще. diff --git a/docs/schema.md b/docs/schema.md new file mode 100644 index 0000000..2e3cd36 --- /dev/null +++ b/docs/schema.md @@ -0,0 +1,40 @@ +# Схема и транзакции + +Лабораторная 5, оценка 4: клиент → HAProxy → FastAPI × 2 → PostgreSQL + Valkey sessions + приватный S3; outbox publisher → Valkey Streams broker → worker. + +| Таблица PostgreSQL | Назначение и ограничения | +| --- | --- | +| users | UUID PK; username UNIQUE; password_hash Argon2id; display_name; created_at | +| chats | UUID PK; title; created_at | +| chat_members | PK (chat_id, user_id); FK на chats и users | +| messages | UUID PK; FK на чат и членство отправителя; text до 4000 code points; created_at | +| message_attachments | FK на messages и attachments; до 10 позиций; UNIQUE(message_id, position); PK(message_id, attachment_id) | +| attachments | UUID PK; FK chat_id и (chat_id, owner_id); UNIQUE object_key; original metadata; status, variants, error_code, created_at/updated_at | +| attachment_jobs | UUID job_id PK; UNIQUE attachment_id/staging_key; version, attempt/budget, lease, retention | +| attachment_outbox | PK (job_id, attempt); available_at, published_at; индекс неопубликованных | +| attachment_job_events | Audit переходов, safe error_code, attempt, время | +| attachment_uploads | UUID PK, UNIQUE object_key, created_at; долговечный журнал незавершённых PUT | +| message_idempotency | PK (user_id, chat_id, key); request_text, request_attachment_ids, message_id FK, expires_at | +| schema_migrations | Имя и SHA-256 применённой миграции; applied_at | + +В PostgreSQL нет bytes оригинала. `object_key=attachments//v1/original` назначает сервер; filename не участвует в ключе. До ответа 202 в таблице attachments есть status=queued, variants=[], error_code=null. Миграция 004 добавляет status, variants, error_code, updated_at; уже существующие ready-вложения lab4 не меняют ключи и остаются доступны. + +`attachment_jobs` хранит один job_id на attachment_id, staging_key, pipeline_version=1, текущую попытку, бюджет, lease_token/lease_until и отметку удаления staging. `attachment_outbox(job_id,attempt)` создаётся в той же транзакции, что metadata/намерение. Dispatcher отмечает published_at после XADD; потеря ответа вызывает безопасный дубль. Отложенный retry получает отдельную строку outbox с available_at. `attachment_job_events` — audit переходов и явных replay без credentials и текста исключений. + +Данные при 202: приватный S3 `attachments/staging/` и COMMIT metadata+job+outbox. Готовность: worker сохраняет неизменённый original и три варианта по детерминированным ключам, затем атомарно фиксирует status=ready и JSON metadata. До ready content отвечает 409. Worker проверяет lease_token под блокировкой строк перед записью результатов. Cleaner не удаляет staging queued/processing; после ready удаляет staging, а failed хранит до STAGING_RETENTION_SECONDS. + +Индексы: UNIQUE users(username); chat_members(user_id, chat_id); messages(chat_id, created_at, id); attachments(chat_id); attachment_uploads(created_at). Первичные/уникальные ключи индексируют поиск ID и object_key. + +Создание чата и его участников атомарно. Неизвестный участник откатывает весь чат. Сообщение проверяет членство и все attachment_ids в том же чате, затем одной транзакцией записывает messages, message_attachments и результат идемпотентности. Составной FK дополнительно запрещает отправителя вне чата. Пустой/пробельный text разрешён только с вложением: это проверяет модель запроса; FK таблицы message_attachments запрещают несуществующие ссылки. Удаления вложений в HTTP API нет. + +Общая advisory transaction lock сериализует область пользователь + POST + путь чата + Idempotency-Key. Сравниваются text и упорядоченный нормализованный список UUID; отсутствие attachment_ids эквивалентно []. Совпадение возвращает исходный 201 и Message, отличие — 409. TTL не меньше 24 часов. Чтение истории использует keyset (created_at,id), без кэша; на следующем запросе видны COMMIT любой реплики. При конкурентных вставках snapshot между страницами нет; запись с более ранними часами может оказаться до уже пройденного cursor. + +Upload сначала полностью принимает ограниченный поток во временный файл и проверяет PNG/JPEG. Затем отдельной транзакцией фиксирует attachment_uploads. В следующей транзакции берётся advisory lock object_key, выполняется S3 PUT, вставляются queued metadata, job и outbox, удаляется журнал. 202 отправляется после COMMIT. S3 и PostgreSQL не образуют распределённую транзакцию. + +Если PUT прошёл, а COMMIT нет, остаётся журнал. Если исход COMMIT неизвестен, немедленного удаления нет: metadata/job могли сохраниться, а удаление журнала атомарно с ними. Cleaner раз в 30 секунд берёт до 100 записей старше grace period (по умолчанию 3600 s, минимум 60 s), пробует ту же advisory lock и повторно проверяет запись. Активная загрузка пропускается; при наличии ready-метаданных удаляется только запись журнала; иначе удаляется S3 object и затем журнал. DELETE объекта идемпотентен. Сбой S3/БД откладывает обработку; следующий проход повторяет её. Это компенсация журналируемых загрузок уровня 4, без полного обхода bucket. + +Миграции 001 и 002 сохранены. Миграция 003 добавляет вложения, связи, журнал и список вложений в ключе повтора; старые сообщения имеют пустой список. DDL выполняет один migrate-service под общей advisory lock; API проверяют версии и checksum. + +Сервис valkey хранит только серверные сессии; отдельный сервис broker хранит Stream. Сессии: session:SHA256(token) → user_id, expires_at, TTL. Включены AOF appendfsync always и постоянный volume. Сырой токен не записывается в evidence. PostgreSQL, Valkey и S3 bucket проверяются /health/ready; /health/live не обращается к зависимостям. + +Основа схемы перенесена из lab4; команды, измерения и ограничения прежней версии — в [REPORT.lab4.md](../REPORT.lab4.md). Исходные отчёты и результаты lab2/lab3 сохранены отдельно. Текущая лабораторная описывается в [REPORT.md](../REPORT.md). diff --git a/evidence/docker-checks.txt b/evidence/docker-checks.txt new file mode 100644 index 0000000..3044d0a --- /dev/null +++ b/evidence/docker-checks.txt @@ -0,0 +1,66 @@ +UTC: 2026-10-22T11:17:22.562634+00:00 +Host: Darwin arm64 Python 3.11.9 +Docker: 29.0.1 aarch64 CPUs=10 RAM=8217448448 +Commit: 9961362 + working tree +Command: ./scripts/check-docker.sh; project: lab2-check-ae280d00db +PASS: start.sh built and started the stack, readiness reached +api image=sha256:0b3dc5056cced642eea5ade11d235dce9f69f0d9e6d63956ce9479f2deaa10a0 started=2026-10-22T11:17:27.057932632Z memory=268435456 nano_cpus=1000000000 +postgres image=sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3 started=2026-10-22T11:17:24.411773881Z memory=536870912 nano_cpus=1000000000 +valkey image=sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 started=2026-10-22T11:17:24.406509756Z memory=134217728 nano_cpus=500000000 +PASS: non-root UID 10001; Python PID 1; healthcheck; named volumes; no API data volume +Проверяем сервер: http://127.0.0.1:61900 +/Users/vanek/Desktop/VNPlabs/VNPlab2/mtusi2026lab2/.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +/Users/vanek/Desktop/VNPlabs/VNPlab2/mtusi2026lab2/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:61900" +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. +test_chat_access (test_api.ApiTests.test_chat_access) ... ok +test_chat_pagination_and_cursor_owner (test_api.ApiTests.test_chat_pagination_and_cursor_owner) ... ok +test_cursor_cannot_be_used_for_another_chat_or_changed (test_api.ApiTests.test_cursor_cannot_be_used_for_another_chat_or_changed) ... ok +test_default_page_size (test_api.ApiTests.test_default_page_size) ... ok +test_duplicate_username (test_api.ApiTests.test_duplicate_username) ... ok +test_framework_errors_and_request_ids (test_api.ApiTests.test_framework_errors_and_request_ids) ... ok +test_identification_and_authentication_order (test_api.ApiTests.test_identification_and_authentication_order) ... ok +test_invalid_chat_members (test_api.ApiTests.test_invalid_chat_members) ... ok +test_invalid_chat_title (test_api.ApiTests.test_invalid_chat_title) ... ok +test_invalid_limits_and_cursors (test_api.ApiTests.test_invalid_limits_and_cursors) ... ok +test_invalid_message_fields (test_api.ApiTests.test_invalid_message_fields) ... ok +test_invalid_user_fields (test_api.ApiTests.test_invalid_user_fields) ... ok +test_malformed_json_and_encoding (test_api.ApiTests.test_malformed_json_and_encoding) ... ok +test_message_pagination_and_changing_limit (test_api.ApiTests.test_message_pagination_and_changing_limit) ... ok +test_unicode_and_text_preservation (test_api.ApiTests.test_unicode_and_text_preservation) ... ok +test_wrong_content_type (test_api.ApiTests.test_wrong_content_type) ... ok +test_simultaneous_registration_of_one_username (test_concurrency.ConcurrencyTests.test_simultaneous_registration_of_one_username) ... ok +test_ten_clients_send_one_hundred_messages (test_concurrency.ConcurrencyTests.test_ten_clients_send_one_hundred_messages) ... ok +test_logout_revokes_only_one_session (test_sessions.SessionTests.test_logout_revokes_only_one_session) ... ok +test_password_validation (test_sessions.SessionTests.test_password_validation) ... ok +test_sql_like_message_is_plain_text (test_sessions.SessionTests.test_sql_like_message_is_plain_text) ... ok +test_unicode_password_with_colon_and_private_response (test_sessions.SessionTests.test_unicode_password_with_colon_and_private_response) ... ok +test_wrong_password_and_basic_only_for_login (test_sessions.SessionTests.test_wrong_password_and_basic_only_for_login) ... ok + +---------------------------------------------------------------------- +Ran 23 tests in 5.743s + +OK +PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages. +PASS: demo.sh registered users, logged in and read two messages +PASS: Argon2id in PostgreSQL; logs do not contain message text, passwords or session tokens +PASS: failed membership insert rolled back chat; migrations run twice without duplicate data +/Users/vanek/Desktop/VNPlabs/VNPlab2/mtusi2026lab2/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:61900" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +Restart api 2026-10-22T11:17:27.057932632Z -> 2026-10-22T11:17:45.460071626Z +Restart postgres 2026-10-22T11:17:24.411773881Z -> 2026-10-22T11:17:44.637353001Z +Restart valkey 2026-10-22T11:17:24.406509756Z -> 2026-10-22T11:17:44.670095293Z +PASS: API, PostgreSQL and Valkey really restarted; previous token still works +PASS: valkey unavailable -> 503 in 0.11s; anonymous -> 401; readiness 503, liveness 200 +API ready +PASS: valkey recovered without a new login +PASS: postgres unavailable -> 503 in 2.01s; anonymous -> 401; readiness 503, liveness 200 +API ready +PASS: postgres recovered without a new login +API ready +PASS: SESSION_TTL_SECONDS=2; valid token expires without logout +PASS: SIGTERM with unfinished request: 5.54s; new requests refused after 0.16s; exit=0; OOM=false +PASS: all lab2 Docker checks completed +PASS: stop.sh removed containers and preserved named volumes +Removed only temporary Compose project and its test volumes: lab2-check-ae280d00db diff --git a/evidence/lab3-checks.json b/evidence/lab3-checks.json new file mode 100644 index 0000000..8c49713 --- /dev/null +++ b/evidence/lab3-checks.json @@ -0,0 +1,1209 @@ +{ + "utc": "2026-10-29T10:13:09.874202+00:00", + "project": "lab3-check-a31f544685", + "base_url": "http://127.0.0.1:58474", + "host": "Darwin arm64", + "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", + "profile": { + "workers": 4, + "pause_seconds": 0.05, + "operation": "POST message then GET history, same-key retry after uncertain POST" + }, + "compose_version": "2.40.3-desktop.1", + "source_commit": "74aa2242515a05192eceb4e3838e3eb4f032f397", + "working_tree": true, + "containers": [ + { + "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", + "instance": "7f6a80b69e32", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:15.412188555Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", + "instance": "f973eaec4d52", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:15.325770597Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", + "instance": "2abbb539f04c", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-10-29T10:13:21.014548544Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "58474" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "a0a498f6df2fa9b5b03b24cb1ef2f3786139a8f744f71a6803e414867c391f94", + "instance": "a0a498f6df2f", + "service": "migrate", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:14.212270221Z", + "running": false, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", + "instance": "653f44c2806d", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-10-29T10:13:11.598535928Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab3-check-a31f544685_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", + "instance": "4aca2308d7a0", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-10-29T10:13:11.59306872Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab3-check-a31f544685_valkey_data" + } + ] + } + ], + "api_tests": "public smoke + 28 student tests passed", + "persistence": { + "before": [ + { + "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", + "instance": "7f6a80b69e32", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:15.412188555Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", + "instance": "f973eaec4d52", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:15.325770597Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", + "instance": "2abbb539f04c", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-10-29T10:13:21.014548544Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "58474" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", + "instance": "653f44c2806d", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-10-29T10:13:11.598535928Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab3-check-a31f544685_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", + "instance": "4aca2308d7a0", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-10-29T10:13:11.59306872Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab3-check-a31f544685_valkey_data" + } + ] + } + ], + "after": [ + { + "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", + "instance": "7f6a80b69e32", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:35.473955634Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", + "instance": "f973eaec4d52", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:35.398920009Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", + "instance": "2abbb539f04c", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-10-29T10:13:35.312310009Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "58474" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", + "instance": "653f44c2806d", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-10-29T10:13:35.342063384Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab3-check-a31f544685_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", + "instance": "4aca2308d7a0", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-10-29T10:13:35.37293605Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab3-check-a31f544685_valkey_data" + } + ] + } + ], + "passed": true, + "idempotency_replay_after_restart": true + }, + "sigterm": { + "instance": "7f6a80b69e32", + "unfinished_request": true, + "shutdown_seconds": 5.393, + "new_connections_refused_seconds": 0.167, + "exit_code": 0, + "oom_killed": false + }, + "dependency_readiness": [ + { + "dependency": "valkey", + "api_live": 200, + "api_ready": 503, + "lb_ready": 503 + }, + { + "dependency": "postgres", + "api_live": 200, + "api_ready": 503, + "lb_ready": 503 + } + ], + "public_failover": { + "passed": true, + "containers_after": [ + { + "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", + "instance": "7f6a80b69e32", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:48.981612751Z", + "running": false, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", + "instance": "f973eaec4d52", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:35.398920009Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + } + ] + }, + "failovers": [ + { + "label": "stop 7f6a80b69e32", + "duration_seconds": 16.009, + "successful_requests": 1576, + "successful_writes": 788, + "errors": 4, + "successful_rps": 98.44, + "first_success_after_action_seconds": { + "POST": 0.014, + "GET": 0.007 + }, + "maximum_success_gap_seconds": { + "POST": 1.957, + "GET": 2.004 + }, + "action_start_seconds": 1.003, + "action_end_seconds": 1.666, + "p50_ms": 7.813, + "p95_ms": 16.393, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "7f6a80b69e32", + "f973eaec4d52" + ], + "per_second": [ + { + "second": 0, + "successes": 121, + "errors": 0 + }, + { + "second": 1, + "successes": 43, + "errors": 0 + }, + { + "second": 2, + "successes": 0, + "errors": 0 + }, + { + "second": 3, + "successes": 73, + "errors": 4 + }, + { + "second": 4, + "successes": 109, + "errors": 0 + }, + { + "second": 5, + "successes": 113, + "errors": 0 + }, + { + "second": 6, + "successes": 116, + "errors": 0 + }, + { + "second": 7, + "successes": 113, + "errors": 0 + }, + { + "second": 8, + "successes": 118, + "errors": 0 + }, + { + "second": 9, + "successes": 112, + "errors": 0 + }, + { + "second": 10, + "successes": 102, + "errors": 0 + }, + { + "second": 11, + "successes": 102, + "errors": 0 + }, + { + "second": 12, + "successes": 110, + "errors": 0 + }, + { + "second": 13, + "successes": 116, + "errors": 0 + }, + { + "second": 14, + "successes": 112, + "errors": 0 + }, + { + "second": 15, + "successes": 116, + "errors": 0 + }, + { + "second": 16, + "successes": 0, + "errors": 0 + } + ], + "error_samples": [ + [ + 3.329044791999877, + "GET", + "GET /api/v1/chats/76736023-9461-41c6-87f7-2286c6b1b7b3/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.363858082999741, + "POST", + "POST /api/v1/chats/76736023-9461-41c6-87f7-2286c6b1b7b3/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.367696166999849, + "POST", + "POST /api/v1/chats/76736023-9461-41c6-87f7-2286c6b1b7b3/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.4346232499997313, + "GET", + "GET /api/v1/chats/76736023-9461-41c6-87f7-2286c6b1b7b3/messages: connection/TLS failure (TimeoutError)" + ] + ], + "action": { + "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", + "instance": "7f6a80b69e32", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:14:23.665074878Z", + "running": false, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + } + }, + { + "label": "stop f973eaec4d52", + "duration_seconds": 16.015, + "successful_requests": 1574, + "successful_writes": 786, + "errors": 4, + "successful_rps": 98.28, + "first_success_after_action_seconds": { + "POST": 0.002, + "GET": 0.008 + }, + "maximum_success_gap_seconds": { + "POST": 2.006, + "GET": 2.006 + }, + "action_start_seconds": 1.005, + "action_end_seconds": 1.713, + "p50_ms": 7.98, + "p95_ms": 17.217, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "7f6a80b69e32", + "f973eaec4d52" + ], + "per_second": [ + { + "second": 0, + "successes": 112, + "errors": 0 + }, + { + "second": 1, + "successes": 43, + "errors": 0 + }, + { + "second": 2, + "successes": 0, + "errors": 0 + }, + { + "second": 3, + "successes": 71, + "errors": 4 + }, + { + "second": 4, + "successes": 117, + "errors": 0 + }, + { + "second": 5, + "successes": 115, + "errors": 0 + }, + { + "second": 6, + "successes": 109, + "errors": 0 + }, + { + "second": 7, + "successes": 116, + "errors": 0 + }, + { + "second": 8, + "successes": 109, + "errors": 0 + }, + { + "second": 9, + "successes": 118, + "errors": 0 + }, + { + "second": 10, + "successes": 115, + "errors": 0 + }, + { + "second": 11, + "successes": 115, + "errors": 0 + }, + { + "second": 12, + "successes": 109, + "errors": 0 + }, + { + "second": 13, + "successes": 105, + "errors": 0 + }, + { + "second": 14, + "successes": 113, + "errors": 0 + }, + { + "second": 15, + "successes": 104, + "errors": 0 + }, + { + "second": 16, + "successes": 3, + "errors": 0 + } + ], + "error_samples": [ + [ + 3.3455931250000503, + "POST", + "POST /api/v1/chats/aba5162e-1b67-4157-9f13-85a2427d7f80/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.349944458000209, + "POST", + "POST /api/v1/chats/aba5162e-1b67-4157-9f13-85a2427d7f80/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.3524864160003744, + "POST", + "POST /api/v1/chats/aba5162e-1b67-4157-9f13-85a2427d7f80/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.41803337500005, + "GET", + "GET /api/v1/chats/aba5162e-1b67-4157-9f13-85a2427d7f80/messages: connection/TLS failure (TimeoutError)" + ] + ], + "action": { + "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", + "instance": "f973eaec4d52", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:13:35.398920009Z", + "running": false, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + } + } + ], + "rolling_update": { + "label": "sequential API replacement", + "duration_seconds": 23.774, + "successful_requests": 2131, + "successful_writes": 1063, + "errors": 11, + "successful_rps": 89.64, + "first_success_after_action_seconds": { + "POST": 0.018, + "GET": 0.004 + }, + "maximum_success_gap_seconds": { + "POST": 1.059, + "GET": 2.034 + }, + "action_start_seconds": 1.005, + "action_end_seconds": 19.63, + "p50_ms": 7.723, + "p95_ms": 13.565, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "244bf0b910eb", + "7f6a80b69e32", + "ca9c7a52da66", + "f973eaec4d52" + ], + "per_second": [ + { + "second": 0, + "successes": 121, + "errors": 0 + }, + { + "second": 1, + "successes": 116, + "errors": 0 + }, + { + "second": 2, + "successes": 109, + "errors": 0 + }, + { + "second": 3, + "successes": 104, + "errors": 0 + }, + { + "second": 4, + "successes": 114, + "errors": 0 + }, + { + "second": 5, + "successes": 33, + "errors": 0 + }, + { + "second": 6, + "successes": 60, + "errors": 4 + }, + { + "second": 7, + "successes": 105, + "errors": 0 + }, + { + "second": 8, + "successes": 117, + "errors": 0 + }, + { + "second": 9, + "successes": 119, + "errors": 0 + }, + { + "second": 10, + "successes": 116, + "errors": 0 + }, + { + "second": 11, + "successes": 115, + "errors": 0 + }, + { + "second": 12, + "successes": 28, + "errors": 0 + }, + { + "second": 13, + "successes": 5, + "errors": 4 + }, + { + "second": 14, + "successes": 74, + "errors": 0 + }, + { + "second": 15, + "successes": 120, + "errors": 0 + }, + { + "second": 16, + "successes": 119, + "errors": 0 + }, + { + "second": 17, + "successes": 115, + "errors": 0 + }, + { + "second": 18, + "successes": 114, + "errors": 0 + }, + { + "second": 19, + "successes": 120, + "errors": 0 + }, + { + "second": 20, + "successes": 116, + "errors": 0 + }, + { + "second": 21, + "successes": 88, + "errors": 0 + }, + { + "second": 22, + "successes": 0, + "errors": 0 + }, + { + "second": 23, + "successes": 3, + "errors": 3 + } + ], + "error_samples": [ + [ + 6.263843457999883, + "POST", + "POST /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [201], received 503" + ], + [ + 6.297545500000069, + "POST", + "POST /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [201], received 503" + ], + [ + 6.300835125000049, + "POST", + "POST /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [201], received 503" + ], + [ + 6.369535666000047, + "GET", + "GET /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [200], received 503" + ], + [ + 13.222746333000032, + "POST", + "POST /api/v1/chats/6c896f9d-2f46-4c63-85f9-59d67053f53b/messages: expected [201], received 503" + ] + ], + "action": { + "before": [ + { + "id": "7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6", + "instance": "7f6a80b69e32", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:14:47.987597625Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b", + "instance": "f973eaec4d52", + "service": "api", + "image": "sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb", + "started_at": "2026-10-29T10:15:11.976704845Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", + "instance": "2abbb539f04c", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-10-29T10:13:35.312310009Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "58474" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", + "instance": "653f44c2806d", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-10-29T10:14:01.922362174Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab3-check-a31f544685_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", + "instance": "4aca2308d7a0", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-10-29T10:13:56.000628546Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab3-check-a31f544685_valkey_data" + } + ] + } + ], + "after": [ + { + "id": "ca9c7a52da66ca3c5ffad5cc6668a0d6d60876862683967b6d5f8e438eb630e0", + "instance": "ca9c7a52da66", + "service": "api", + "image": "sha256:3e102cd9a2c1e81e95aed4a0d4de1ac94352370595fced350257813f7a07bef8", + "started_at": "2026-10-29T10:15:24.374360503Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "244bf0b910eb2fa37937f8f0756a1162e9b339df076f09f4cff90b5c357037a6", + "instance": "244bf0b910eb", + "service": "api", + "image": "sha256:3e102cd9a2c1e81e95aed4a0d4de1ac94352370595fced350257813f7a07bef8", + "started_at": "2026-10-29T10:15:31.325585882Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "2abbb539f04ceb95a2d783c1326fd65d7083216f5846e73ca3f253237fa3c6e1", + "instance": "2abbb539f04c", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-10-29T10:13:35.312310009Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "58474" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "653f44c2806d6c0dce3f9695ea8cde24648091b2516976aace0bece9906b064d", + "instance": "653f44c2806d", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-10-29T10:14:01.922362174Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab3-check-a31f544685_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "4aca2308d7a06f00722910ca2c010a9f9a55d6c9c8a2a02af1c7502ad8cafe2e", + "instance": "4aca2308d7a0", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-10-29T10:13:56.000628546Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab3-check-a31f544685_valkey_data" + } + ] + } + ] + } + }, + "passed": true +} diff --git a/evidence/lab3-checks.txt b/evidence/lab3-checks.txt new file mode 100644 index 0000000..27ec14e --- /dev/null +++ b/evidence/lab3-checks.txt @@ -0,0 +1,403 @@ +Command: ./scripts/check-docker.sh; project: lab3-check-a31f544685 +Docker: 29.0.1 aarch64 CPUs=10 RAM=8217448448 +Local .env and secrets are ready; values are not printed +#1 [internal] load local bake definitions +#1 reading from stdin 1.05kB done +#1 DONE 0.0s + +#2 [migrate internal] load build definition from Dockerfile +#2 transferring dockerfile: 733B done +#2 DONE 0.0s + +#3 [api internal] load metadata for docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#3 DONE 0.0s + +#4 [api internal] load .dockerignore +#4 transferring context: 167B done +#4 DONE 0.0s + +#5 [migrate internal] load build context +#5 transferring context: 809B done +#5 DONE 0.0s + +#6 [migrate 1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 done +#6 DONE 0.0s + +#7 [migrate 2/8] WORKDIR /app +#7 CACHED + +#8 [migrate 3/8] COPY requirements.txt . +#8 CACHED + +#9 [migrate 4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#9 CACHED + +#10 [migrate 6/8] COPY src/ ./src/ +#10 CACHED + +#11 [migrate 7/8] COPY migrations/ ./migrations/ +#11 CACHED + +#12 [migrate 5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#12 CACHED + +#13 [migrate 8/8] COPY research/ ./research/ +#13 CACHED + +#14 [api] exporting to image +#14 exporting layers done +#14 exporting manifest sha256:06163e45d3676c7cf4f7a1c7bfe108ed3b98559ebb485c7a09d1e87e6f418d17 done +#14 exporting config sha256:3fed7460e9510eef7b2928f8d600d06d5d65329f74d820beac13809476b11398 done +#14 exporting attestation manifest sha256:352319cde788733f2f237a2a78204630c542e7ca906cff14c902046e8c1d2e5b 0.0s done +#14 exporting manifest list sha256:848e0c76f1678f35dd1b483e7c54552466999551ba7e77c0062bf346671e2902 done +#14 naming to docker.io/library/mtusi-messenger:lab3 done +#14 unpacking to docker.io/library/mtusi-messenger:lab3 done +#14 DONE 0.1s + +#15 [migrate] exporting to image +#15 exporting layers done +#15 exporting manifest sha256:a59568d34e2c5e8b4d23308342601fdaf8c07e78ec3065878679a067c082e952 done +#15 exporting config sha256:b67b805ba1b7dd90f0a3e8444455488fe8c7f4118f14d671501dd63e85b59c97 done +#15 exporting attestation manifest sha256:b1853c07ddac46c4b398b475052f334aa2753c2fe971e0e4fb2517b0e3e13656 0.0s done +#15 exporting manifest list sha256:eff1abdd662eb60e2741e6ce6b8f364467d2817dafeef9754518247124a780eb done +#15 naming to docker.io/library/mtusi-messenger:lab3 done +#15 unpacking to docker.io/library/mtusi-messenger:lab3 done +#15 DONE 0.1s + +#16 [api] resolving provenance for metadata file +#16 DONE 0.0s + +#17 [migrate] resolving provenance for metadata file +#17 DONE 0.0s + mtusi-messenger:lab3 Built + mtusi-messenger:lab3 Built + Network lab3-check-a31f544685_default Creating + Network lab3-check-a31f544685_default Created + Volume lab3-check-a31f544685_postgres_data Creating + Volume lab3-check-a31f544685_postgres_data Created + Volume lab3-check-a31f544685_valkey_data Creating + Volume lab3-check-a31f544685_valkey_data Created + Container lab3-check-a31f544685-valkey-1 Creating + Container lab3-check-a31f544685-postgres-1 Creating + Container lab3-check-a31f544685-valkey-1 Created + Container lab3-check-a31f544685-postgres-1 Created + Container lab3-check-a31f544685-migrate-1 Creating + Container lab3-check-a31f544685-migrate-1 Created + Container lab3-check-a31f544685-api-2 Creating + Container lab3-check-a31f544685-api-1 Creating + Container lab3-check-a31f544685-api-1 Created + Container lab3-check-a31f544685-api-2 Created + Container lab3-check-a31f544685-lb-1 Creating + Container lab3-check-a31f544685-lb-1 Created + Container lab3-check-a31f544685-valkey-1 Starting + Container lab3-check-a31f544685-postgres-1 Starting + Container lab3-check-a31f544685-valkey-1 Started + Container lab3-check-a31f544685-postgres-1 Started + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Starting + Container lab3-check-a31f544685-migrate-1 Started + Container lab3-check-a31f544685-migrate-1 Waiting + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-valkey-1 Waiting + Container lab3-check-a31f544685-valkey-1 Healthy + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Exited + Container lab3-check-a31f544685-api-2 Starting + Container lab3-check-a31f544685-api-2 Started + Container lab3-check-a31f544685-api-1 Starting + Container lab3-check-a31f544685-api-1 Started + Container lab3-check-a31f544685-api-2 Waiting + Container lab3-check-a31f544685-api-1 Waiting + Container lab3-check-a31f544685-api-2 Healthy + Container lab3-check-a31f544685-api-1 Healthy + Container lab3-check-a31f544685-lb-1 Starting + Container lab3-check-a31f544685-lb-1 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 +PASS: 2 real API containers, CPU/RAM limits, API non-root and no public ports; only LB published +Проверяем сервер: http://127.0.0.1:58474 +/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:58474" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. +test_chat_access (test_api.ApiTests.test_chat_access) ... ok +test_chat_pagination_and_cursor_owner (test_api.ApiTests.test_chat_pagination_and_cursor_owner) ... ok +test_cursor_cannot_be_used_for_another_chat_or_changed (test_api.ApiTests.test_cursor_cannot_be_used_for_another_chat_or_changed) ... ok +test_default_page_size (test_api.ApiTests.test_default_page_size) ... ok +test_duplicate_username (test_api.ApiTests.test_duplicate_username) ... ok +test_framework_errors_and_request_ids (test_api.ApiTests.test_framework_errors_and_request_ids) ... ok +test_identification_and_authentication_order (test_api.ApiTests.test_identification_and_authentication_order) ... ok +test_invalid_chat_members (test_api.ApiTests.test_invalid_chat_members) ... ok +test_invalid_chat_title (test_api.ApiTests.test_invalid_chat_title) ... ok +test_invalid_limits_and_cursors (test_api.ApiTests.test_invalid_limits_and_cursors) ... ok +test_invalid_message_fields (test_api.ApiTests.test_invalid_message_fields) ... ok +test_invalid_user_fields (test_api.ApiTests.test_invalid_user_fields) ... ok +test_malformed_json_and_encoding (test_api.ApiTests.test_malformed_json_and_encoding) ... ok +test_message_pagination_and_changing_limit (test_api.ApiTests.test_message_pagination_and_changing_limit) ... ok +test_unicode_and_text_preservation (test_api.ApiTests.test_unicode_and_text_preservation) ... ok +test_wrong_content_type (test_api.ApiTests.test_wrong_content_type) ... ok +test_simultaneous_registration_of_one_username (test_concurrency.ConcurrencyTests.test_simultaneous_registration_of_one_username) ... ok +test_ten_clients_send_one_hundred_messages (test_concurrency.ConcurrencyTests.test_ten_clients_send_one_hundred_messages) ... ok +test_conflict_and_semantic_json_replay (test_idempotency.IdempotencyTests.test_conflict_and_semantic_json_replay) ... ok +test_key_format_and_missing_key (test_idempotency.IdempotencyTests.test_key_format_and_missing_key) ... ok +test_parallel_replay_across_replicas (test_idempotency.IdempotencyTests.test_parallel_replay_across_replicas) ... ok +test_replay_rechecks_session_and_access (test_idempotency.IdempotencyTests.test_replay_rechecks_session_and_access) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_logout_revokes_only_one_session (test_sessions.SessionTests.test_logout_revokes_only_one_session) ... ok +test_password_validation (test_sessions.SessionTests.test_password_validation) ... ok +test_sql_like_message_is_plain_text (test_sessions.SessionTests.test_sql_like_message_is_plain_text) ... ok +test_unicode_password_with_colon_and_private_response (test_sessions.SessionTests.test_unicode_password_with_colon_and_private_response) ... ok +test_wrong_password_and_basic_only_for_login (test_sessions.SessionTests.test_wrong_password_and_basic_only_for_login) ... ok + +---------------------------------------------------------------------- +Ran 28 tests in 6.783s + +OK +PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages. + Container lab3-check-a31f544685-postgres-1 Running + Container lab3-check-a31f544685-postgres-1 Running +PASS: migrations only in one-shot service; repeated execution preserves data +/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:58474" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: both API, LB, PostgreSQL, Valkey actually restarted; session and confirmed data preserved +PASS: SIGTERM with unfinished request: {"instance": "7f6a80b69e32", "unfinished_request": true, "shutdown_seconds": 5.393, "new_connections_refused_seconds": 0.167, "exit_code": 0, "oom_killed": false} + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Starting + Container lab3-check-a31f544685-migrate-1 Started + Container lab3-check-a31f544685-valkey-1 Waiting + Container lab3-check-a31f544685-migrate-1 Waiting + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-valkey-1 Healthy + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Exited + Container lab3-check-a31f544685-api-1 Starting + Container lab3-check-a31f544685-api-1 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 + Container lab3-check-a31f544685-valkey-1 Stopping + Container lab3-check-a31f544685-valkey-1 Stopped +PASS: valkey outage: API live=200, ready=503, LB ready=503 + Container lab3-check-a31f544685-valkey-1 Starting + Container lab3-check-a31f544685-valkey-1 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 + Container lab3-check-a31f544685-postgres-1 Stopping + Container lab3-check-a31f544685-postgres-1 Stopped +PASS: postgres outage: API live=200, ready=503, LB ready=503 + Container lab3-check-a31f544685-postgres-1 Starting + Container lab3-check-a31f544685-postgres-1 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 +/Users/vanek/Desktop/VNPlabs/VNPlab3/mtusi2026lab3/.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:58474" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 113, "failed_attempts": 1, "first_recovery_seconds": 2.452} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Starting + Container lab3-check-a31f544685-migrate-1 Started + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-valkey-1 Waiting + Container lab3-check-a31f544685-migrate-1 Waiting + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-valkey-1 Healthy + Container lab3-check-a31f544685-migrate-1 Exited + Container lab3-check-a31f544685-api-1 Starting + Container lab3-check-a31f544685-api-1 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 +Stopped API: 7f6a80b69e32 container=7f6a80b69e3297a4de72e756a6c9d867a108930c1ca0e57922b3821c26fd03e6 +{"label": "stop 7f6a80b69e32", "duration_seconds": 16.009, "successful_requests": 1576, "successful_writes": 788, "errors": 4, "successful_rps": 98.44, "first_success_after_action_seconds": {"POST": 0.014, "GET": 0.007}, "maximum_success_gap_seconds": {"POST": 1.957, "GET": 2.004}, "action_start_seconds": 1.003, "action_end_seconds": 1.666, "p50_ms": 7.813, "p95_ms": 16.393, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["7f6a80b69e32", "f973eaec4d52"]} + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Starting + Container lab3-check-a31f544685-migrate-1 Started + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-valkey-1 Waiting + Container lab3-check-a31f544685-migrate-1 Waiting + Container lab3-check-a31f544685-valkey-1 Healthy + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Exited + Container lab3-check-a31f544685-api-1 Starting + Container lab3-check-a31f544685-api-1 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 +Stopped API: f973eaec4d52 container=f973eaec4d5290eb73cd96949877ccb8df1213523fce0ec05c6b40c1136bb31b +{"label": "stop f973eaec4d52", "duration_seconds": 16.015, "successful_requests": 1574, "successful_writes": 786, "errors": 4, "successful_rps": 98.28, "first_success_after_action_seconds": {"POST": 0.002, "GET": 0.008}, "maximum_success_gap_seconds": {"POST": 2.006, "GET": 2.006}, "action_start_seconds": 1.005, "action_end_seconds": 1.713, "p50_ms": 7.98, "p95_ms": 17.217, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["7f6a80b69e32", "f973eaec4d52"]} + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Starting + Container lab3-check-a31f544685-migrate-1 Started + Container lab3-check-a31f544685-postgres-1 Waiting + Container lab3-check-a31f544685-valkey-1 Waiting + Container lab3-check-a31f544685-migrate-1 Waiting + Container lab3-check-a31f544685-valkey-1 Healthy + Container lab3-check-a31f544685-postgres-1 Healthy + Container lab3-check-a31f544685-migrate-1 Exited + Container lab3-check-a31f544685-api-2 Starting + Container lab3-check-a31f544685-api-2 Started +API ready through LB: 7f6a80b69e32, f973eaec4d52 +Local .env and secrets are ready; values are not printed +#1 [internal] load local bake definitions +#1 reading from stdin 1.03kB done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 733B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 167B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 809B done +#5 DONE 0.0s + +#6 [1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 0.0s done +#6 DONE 0.0s + +#7 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#7 CACHED + +#8 [3/8] COPY requirements.txt . +#8 CACHED + +#9 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#9 CACHED + +#10 [6/8] COPY src/ ./src/ +#10 CACHED + +#11 [2/8] WORKDIR /app +#11 CACHED + +#12 [7/8] COPY migrations/ ./migrations/ +#12 CACHED + +#13 [8/8] COPY research/ ./research/ +#13 CACHED + +#14 exporting to image +#14 exporting layers done +#14 exporting manifest sha256:06163e45d3676c7cf4f7a1c7bfe108ed3b98559ebb485c7a09d1e87e6f418d17 done +#14 exporting config sha256:3fed7460e9510eef7b2928f8d600d06d5d65329f74d820beac13809476b11398 done +#14 exporting attestation manifest sha256:3a88ce0975ddcbab53c3ecb185da08ef114e818e9db952983e997cd47add67bf done +#14 exporting manifest list sha256:433a8a3293202a92f092574052b474e8373c0d58649cbcf1c7ebc1af17933d3c done +#14 naming to docker.io/library/mtusi-messenger:lab3 done +#14 unpacking to docker.io/library/mtusi-messenger:lab3 done +#14 DONE 0.1s + +#15 resolving provenance for metadata file +#15 DONE 0.0s + mtusi-messenger:lab3 Built +Local .env and secrets are ready; values are not printed + Container lab3-check-a31f544685-postgres-1 Running +#1 [internal] load local bake definitions +#1 reading from stdin 576B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 733B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 167B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 809B done +#5 DONE 0.0s + +#6 [1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 done +#6 DONE 0.0s + +#7 [3/8] COPY requirements.txt . +#7 CACHED + +#8 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#8 CACHED + +#9 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#9 CACHED + +#10 [6/8] COPY src/ ./src/ +#10 CACHED + +#11 [7/8] COPY migrations/ ./migrations/ +#11 CACHED + +#12 [2/8] WORKDIR /app +#12 CACHED + +#13 [8/8] COPY research/ ./research/ +#13 CACHED + +#14 exporting to image +#14 exporting layers done +#14 exporting manifest sha256:a59568d34e2c5e8b4d23308342601fdaf8c07e78ec3065878679a067c082e952 done +#14 exporting config sha256:b67b805ba1b7dd90f0a3e8444455488fe8c7f4118f14d671501dd63e85b59c97 done +#14 exporting attestation manifest sha256:25ff7934ca5161da9b6c345af61b6a10b452580e4d349b35939d32feb66664ea done +#14 exporting manifest list sha256:3e102cd9a2c1e81e95aed4a0d4de1ac94352370595fced350257813f7a07bef8 done +#14 naming to docker.io/library/mtusi-messenger:lab3 done +#14 unpacking to docker.io/library/mtusi-messenger:lab3 done +#14 DONE 0.0s + +#15 resolving provenance for metadata file +#15 DONE 0.0s +Migrations: already applied +API ready through LB: 7f6a80b69e32, f973eaec4d52 +Replacing API: 7f6a80b69e32 + Container lab3-check-a31f544685-api-2 Running + Container lab3-check-a31f544685-api-3 Creating + Container lab3-check-a31f544685-api-3 Created + Container lab3-check-a31f544685-api-3 Starting + Container lab3-check-a31f544685-api-3 Started +API ready through LB: ca9c7a52da66, f973eaec4d52 +Replacing API: f973eaec4d52 + Container lab3-check-a31f544685-api-3 Running + Container lab3-check-a31f544685-api-4 Creating + Container lab3-check-a31f544685-api-4 Created + Container lab3-check-a31f544685-api-4 Starting + Container lab3-check-a31f544685-api-4 Started +API ready through LB: 244bf0b910eb, ca9c7a52da66 +PASS: all API containers replaced sequentially; stores and LB kept running +{"label": "sequential API replacement", "duration_seconds": 23.774, "successful_requests": 2131, "successful_writes": 1063, "errors": 11, "successful_rps": 89.64, "first_success_after_action_seconds": {"POST": 0.018, "GET": 0.004}, "maximum_success_gap_seconds": {"POST": 1.059, "GET": 2.034}, "action_start_seconds": 1.005, "action_end_seconds": 19.63, "p50_ms": 7.723, "p95_ms": 13.565, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["244bf0b910eb", "7f6a80b69e32", "ca9c7a52da66", "f973eaec4d52"]} +PASS: all lab3 grade-4 checks completed + Container lab3-check-a31f544685-lb-1 Stopping + Container lab3-check-a31f544685-lb-1 Stopped + Container lab3-check-a31f544685-lb-1 Removing + Container lab3-check-a31f544685-lb-1 Removed + Container lab3-check-a31f544685-api-3 Stopping + Container lab3-check-a31f544685-api-4 Stopping + Container lab3-check-a31f544685-api-4 Stopped + Container lab3-check-a31f544685-api-4 Removing + Container lab3-check-a31f544685-api-4 Removed + Container lab3-check-a31f544685-api-3 Stopped + Container lab3-check-a31f544685-api-3 Removing + Container lab3-check-a31f544685-api-3 Removed + Container lab3-check-a31f544685-migrate-1 Stopping + Container lab3-check-a31f544685-valkey-1 Stopping + Container lab3-check-a31f544685-migrate-1 Stopped + Container lab3-check-a31f544685-migrate-1 Removing + Container lab3-check-a31f544685-migrate-1 Removed + Container lab3-check-a31f544685-postgres-1 Stopping + Container lab3-check-a31f544685-valkey-1 Stopped + Container lab3-check-a31f544685-valkey-1 Removing + Container lab3-check-a31f544685-valkey-1 Removed + Container lab3-check-a31f544685-postgres-1 Stopped + Container lab3-check-a31f544685-postgres-1 Removing + Container lab3-check-a31f544685-postgres-1 Removed + Volume lab3-check-a31f544685_valkey_data Removing + Network lab3-check-a31f544685_default Removing + Volume lab3-check-a31f544685_postgres_data Removing + Volume lab3-check-a31f544685_valkey_data Removed + Volume lab3-check-a31f544685_postgres_data Removed + Network lab3-check-a31f544685_default Removed +Removed only temporary test containers and volumes: lab3-check-a31f544685 diff --git a/evidence/lab4-checks.json b/evidence/lab4-checks.json new file mode 100644 index 0000000..f410993 --- /dev/null +++ b/evidence/lab4-checks.json @@ -0,0 +1,2269 @@ +{ + "utc": "2026-11-06T07:29:29.234494+00:00", + "project": "lab4-check-5aad8e0065", + "base_url": "http://127.0.0.1:57722", + "host": "Darwin arm64", + "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", + "profile": { + "workers": 4, + "pause_seconds": 0.05, + "operation": "POST message then GET history, same-key retry after uncertain POST" + }, + "compose_version": "2.40.3-desktop.1", + "source_commit": "c43d3490a07ad939e4e4748863589b73c84ad05a", + "working_tree": true, + "containers": [ + { + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:57.621726877Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:57.502184793Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", + "service": "cleanup", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:57.50177996Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-11-06T07:30:03.264509213Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "57722" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "533e4e511e0441fa577519bbca2a58bc4ceede3ab1660f4c8b026a54b69829f3", + "instance": "533e4e511e04", + "service": "migrate", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:33.838693505Z", + "running": false, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-11-06T07:29:31.155947879Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab4-check-5aad8e0065_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", + "service": "s3", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:29:53.783891208Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" + } + ] + }, + { + "id": "813a7d0f6af841b2ed44a684a33baddbbe41405b08d4bdd499ee7ed12b9eccc1", + "instance": "813a7d0f6af8", + "service": "s3-bootstrap", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:56.406424376Z", + "running": false, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_root_user", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_root_password", + "name": null + } + ] + }, + { + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", + "service": "s3-storage", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:29:31.154902962Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_s3_data" + } + ] + }, + { + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-11-06T07:29:31.157226212Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_valkey_data" + } + ] + } + ], + "api_tests": "public smoke + student unittest suite passed", + "attachments": { + "private_bucket": { + "anonymous_get": 403, + "create_bucket": "AccessDenied", + "write_outside_prefix": "AccessDenied", + "admin_ports_unreachable": [ + 9333, + 8888, + 8080 + ], + "storage_network_isolated": true, + "bootstrap_runs": 2 + }, + "upload_replica_failover": { + "uploaded_by": "ca1f62dfc015", + "stopped_container": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "downloaded_by": "19a474c69210", + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "recovery_seconds": 3.535 + }, + "interrupted_body": { + "metadata_or_journal_created": false + }, + "compensation": { + "failure_after_put": 503, + "orphan_recorded": true, + "fresh_kept": true, + "active_locked_kept": true, + "expired_removed": true, + "ready_kept": true, + "repeat_safe": true, + "automatic_worker": true + }, + "s3_outage": { + "live": 200, + "ready": 503, + "download": 503, + "upload": 503, + "original_preserved_after_recovery": true + }, + "parallel_uploads": { + "concurrency": 10, + "successful": 10, + "errors": 0, + "dimensions": [ + 1600, + 1200 + ], + "size_bytes": 5769335, + "sha256": "b43f2e8293c15183ae478f3bc43cd262e1fa2c532e1c2cfbabceaf3566b4a0d1", + "duration_seconds": 0.914, + "successful_rps": 10.939, + "p50_ms": 610.113, + "p95_ms": 904.09, + "memory_sample_interval_seconds": 0.02, + "memory_includes_monitor_process": true, + "memory": [ + { + "baseline_bytes": 199331840, + "sample_peak_bytes": 215191552, + "lifetime_peak_bytes": 274231296, + "samples": [ + [ + 0.0, + 199098368 + ], + [ + 0.021, + 201449472 + ], + [ + 0.043, + 201371648 + ], + [ + 0.064, + 201367552 + ], + [ + 0.085, + 201625600 + ], + [ + 0.108, + 202936320 + ], + [ + 0.13, + 202309632 + ], + [ + 0.151, + 188710912 + ], + [ + 0.172, + 189378560 + ], + [ + 0.193, + 190976000 + ], + [ + 0.214, + 192249856 + ], + [ + 0.235, + 193933312 + ], + [ + 0.255, + 195235840 + ], + [ + 0.276, + 196837376 + ], + [ + 0.297, + 197255168 + ], + [ + 0.318, + 198422528 + ], + [ + 0.339, + 198475776 + ], + [ + 0.36, + 200818688 + ], + [ + 0.381, + 201170944 + ], + [ + 0.402, + 204890112 + ], + [ + 0.423, + 206053376 + ], + [ + 0.444, + 206901248 + ], + [ + 0.465, + 207024128 + ], + [ + 0.486, + 207085568 + ], + [ + 0.507, + 209682432 + ], + [ + 0.531, + 214925312 + ], + [ + 0.552, + 215191552 + ], + [ + 0.573, + 215191552 + ], + [ + 0.594, + 211496960 + ], + [ + 0.615, + 211415040 + ], + [ + 0.636, + 211611648 + ], + [ + 0.657, + 211410944 + ], + [ + 0.678, + 206876672 + ], + [ + 0.699, + 207892480 + ], + [ + 0.72, + 207699968 + ], + [ + 0.742, + 208068608 + ], + [ + 0.763, + 202301440 + ], + [ + 0.784, + 207642624 + ], + [ + 0.805, + 207413248 + ], + [ + 0.826, + 207593472 + ], + [ + 0.847, + 207593472 + ], + [ + 0.868, + 204062720 + ], + [ + 0.889, + 203718656 + ], + [ + 0.91, + 205983744 + ], + [ + 0.931, + 203411456 + ], + [ + 0.952, + 203411456 + ], + [ + 0.973, + 200871936 + ], + [ + 0.996, + 197525504 + ], + [ + 1.019, + 197496832 + ], + [ + 1.04, + 197496832 + ] + ], + "instance": "19a474c69210", + "limit_bytes": 402653184 + }, + { + "baseline_bytes": 67555328, + "sample_peak_bytes": 103378944, + "lifetime_peak_bytes": 105148416, + "samples": [ + [ + 0.0, + 67555328 + ], + [ + 0.02, + 68780032 + ], + [ + 0.041, + 71090176 + ], + [ + 0.062, + 71782400 + ], + [ + 0.083, + 73428992 + ], + [ + 0.104, + 74698752 + ], + [ + 0.125, + 76619776 + ], + [ + 0.146, + 77070336 + ], + [ + 0.167, + 78118912 + ], + [ + 0.189, + 79024128 + ], + [ + 0.209, + 80576512 + ], + [ + 0.23, + 81674240 + ], + [ + 0.251, + 84914176 + ], + [ + 0.272, + 87511040 + ], + [ + 0.293, + 88616960 + ], + [ + 0.315, + 89174016 + ], + [ + 0.335, + 89919488 + ], + [ + 0.356, + 89976832 + ], + [ + 0.381, + 96006144 + ], + [ + 0.402, + 96436224 + ], + [ + 0.422, + 95522816 + ], + [ + 0.444, + 93450240 + ], + [ + 0.465, + 98467840 + ], + [ + 0.485, + 103378944 + ], + [ + 0.507, + 102744064 + ], + [ + 0.527, + 102805504 + ], + [ + 0.549, + 100847616 + ], + [ + 0.571, + 102080512 + ], + [ + 0.592, + 102064128 + ], + [ + 0.614, + 102064128 + ], + [ + 0.635, + 102318080 + ], + [ + 0.655, + 103100416 + ], + [ + 0.676, + 100167680 + ], + [ + 0.722, + 100024320 + ], + [ + 0.743, + 100548608 + ], + [ + 0.764, + 101707776 + ], + [ + 0.785, + 87347200 + ], + [ + 0.822, + 81424384 + ], + [ + 0.843, + 83718144 + ], + [ + 0.866, + 83685376 + ], + [ + 0.887, + 83664896 + ], + [ + 0.909, + 83668992 + ], + [ + 0.93, + 78086144 + ] + ], + "instance": "ca1f62dfc015", + "limit_bytes": 402653184 + } + ], + "all_original_checksums_verified": true, + "oom_killed": false + } + }, + "persistence": { + "before": [ + { + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:29:57.621726877Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:30:34.948927714Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", + "service": "cleanup", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:01.671594045Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-11-06T07:30:03.264509213Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "57722" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-11-06T07:29:31.155947879Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab4-check-5aad8e0065_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", + "service": "s3", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:30:53.867775792Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" + } + ] + }, + { + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", + "service": "s3-storage", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:29:31.154902962Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_s3_data" + } + ] + }, + { + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-11-06T07:29:31.157226212Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_valkey_data" + } + ] + } + ], + "after": [ + { + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:09.786276466Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:10.013737549Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", + "service": "cleanup", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:17.411326011Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-11-06T07:31:09.542736424Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "57722" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-11-06T07:31:09.573287299Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab4-check-5aad8e0065_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", + "service": "s3", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:31:09.608616716Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" + } + ] + }, + { + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", + "service": "s3-storage", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:31:17.386629011Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_s3_data" + } + ] + }, + { + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-11-06T07:31:09.641720258Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_valkey_data" + } + ] + } + ], + "passed": true, + "idempotency_replay_after_restart": true + }, + "sigterm": { + "instance": "19a474c69210", + "unfinished_request": true, + "shutdown_seconds": 5.523, + "new_connections_refused_seconds": 0.185, + "exit_code": 0, + "oom_killed": false + }, + "dependency_readiness": [ + { + "dependency": "valkey", + "api_live": 200, + "api_ready": 503, + "lb_ready": 503 + }, + { + "dependency": "postgres", + "api_live": 200, + "api_ready": 503, + "lb_ready": 503 + }, + { + "dependency": "s3", + "api_live": 200, + "api_ready": 503, + "lb_ready": 503 + } + ], + "postgres_pause": { + "sql_without_server_timeout": { + "elapsed_seconds": 2.0, + "connection_closed": true + }, + "direct_ready": { + "status": 503, + "elapsed_seconds": 2.013 + }, + "direct_live": { + "status": 200, + "elapsed_seconds": 0.006 + }, + "repeated_ready": [ + { + "status": 503, + "elapsed_seconds": 2.107 + }, + { + "status": 503, + "elapsed_seconds": 2.097 + }, + { + "status": 503, + "elapsed_seconds": 2.094 + }, + { + "status": 503, + "elapsed_seconds": 2.012 + }, + { + "status": 503, + "elapsed_seconds": 2.009 + }, + { + "status": 503, + "elapsed_seconds": 2.095 + }, + { + "status": 503, + "elapsed_seconds": 2.094 + }, + { + "status": 503, + "elapsed_seconds": 2.099 + }, + { + "status": 503, + "elapsed_seconds": 2.007 + }, + { + "status": 503, + "elapsed_seconds": 2.102 + } + ], + "ready_after_12_seconds": { + "status": 503, + "elapsed_seconds": 2.008 + }, + "live_after_repeated_requests": { + "status": 200, + "elapsed_seconds": 0.006 + }, + "paused_seconds": 14.572, + "recovered_ready": { + "status": 200, + "elapsed_seconds": 0.009 + }, + "recovery_seconds": 1.975, + "api_not_restarted": true, + "existing_session_and_message_preserved": true, + "passed": true + }, + "public_failover": { + "passed": true, + "containers_after": [ + { + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:50.739701388Z", + "running": false, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:10.013737549Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + } + ] + }, + "failovers": [ + { + "label": "stop 19a474c69210", + "duration_seconds": 16.012, + "successful_requests": 1420, + "successful_writes": 709, + "errors": 4, + "successful_rps": 88.68, + "first_success_after_action_seconds": { + "POST": 0.051, + "GET": 0.004 + }, + "maximum_success_gap_seconds": { + "POST": 2.002, + "GET": 1.995 + }, + "action_start_seconds": 1.006, + "action_end_seconds": 1.77, + "p50_ms": 10.62, + "p95_ms": 24.67, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "19a474c69210", + "ca1f62dfc015" + ], + "per_second": [ + { + "second": 0, + "successes": 111, + "errors": 0 + }, + { + "second": 1, + "successes": 28, + "errors": 0 + }, + { + "second": 2, + "successes": 0, + "errors": 0 + }, + { + "second": 3, + "successes": 70, + "errors": 4 + }, + { + "second": 4, + "successes": 101, + "errors": 0 + }, + { + "second": 5, + "successes": 94, + "errors": 0 + }, + { + "second": 6, + "successes": 106, + "errors": 0 + }, + { + "second": 7, + "successes": 100, + "errors": 0 + }, + { + "second": 8, + "successes": 97, + "errors": 0 + }, + { + "second": 9, + "successes": 107, + "errors": 0 + }, + { + "second": 10, + "successes": 93, + "errors": 0 + }, + { + "second": 11, + "successes": 97, + "errors": 0 + }, + { + "second": 12, + "successes": 101, + "errors": 0 + }, + { + "second": 13, + "successes": 107, + "errors": 0 + }, + { + "second": 14, + "successes": 107, + "errors": 0 + }, + { + "second": 15, + "successes": 101, + "errors": 0 + }, + { + "second": 16, + "successes": 0, + "errors": 0 + } + ], + "error_samples": [ + [ + 3.2868632079917006, + "POST", + "POST /api/v1/chats/ba1880c0-8f0e-4d7d-ac12-a00beb77fb97/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.2872469169960823, + "POST", + "POST /api/v1/chats/ba1880c0-8f0e-4d7d-ac12-a00beb77fb97/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.2959944579924922, + "POST", + "POST /api/v1/chats/ba1880c0-8f0e-4d7d-ac12-a00beb77fb97/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.3688164169871015, + "GET", + "GET /api/v1/chats/ba1880c0-8f0e-4d7d-ac12-a00beb77fb97/messages: connection/TLS failure (TimeoutError)" + ] + ], + "action": { + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:32:49.121677512Z", + "running": false, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + } + }, + { + "label": "stop ca1f62dfc015", + "duration_seconds": 16.008, + "successful_requests": 1466, + "successful_writes": 732, + "errors": 4, + "successful_rps": 91.58, + "first_success_after_action_seconds": { + "POST": 0.018, + "GET": 0.028 + }, + "maximum_success_gap_seconds": { + "POST": 2.002, + "GET": 1.999 + }, + "action_start_seconds": 1.002, + "action_end_seconds": 1.832, + "p50_ms": 9.397, + "p95_ms": 18.616, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "19a474c69210", + "ca1f62dfc015" + ], + "per_second": [ + { + "second": 0, + "successes": 112, + "errors": 0 + }, + { + "second": 1, + "successes": 43, + "errors": 0 + }, + { + "second": 2, + "successes": 0, + "errors": 0 + }, + { + "second": 3, + "successes": 52, + "errors": 4 + }, + { + "second": 4, + "successes": 108, + "errors": 0 + }, + { + "second": 5, + "successes": 103, + "errors": 0 + }, + { + "second": 6, + "successes": 106, + "errors": 0 + }, + { + "second": 7, + "successes": 111, + "errors": 0 + }, + { + "second": 8, + "successes": 95, + "errors": 0 + }, + { + "second": 9, + "successes": 103, + "errors": 0 + }, + { + "second": 10, + "successes": 103, + "errors": 0 + }, + { + "second": 11, + "successes": 106, + "errors": 0 + }, + { + "second": 12, + "successes": 102, + "errors": 0 + }, + { + "second": 13, + "successes": 109, + "errors": 0 + }, + { + "second": 14, + "successes": 109, + "errors": 0 + }, + { + "second": 15, + "successes": 103, + "errors": 0 + }, + { + "second": 16, + "successes": 1, + "errors": 0 + } + ], + "error_samples": [ + [ + 3.398277083004359, + "POST", + "POST /api/v1/chats/a2185e1c-f008-4190-ae30-86115db5f6df/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.405396166999708, + "POST", + "POST /api/v1/chats/a2185e1c-f008-4190-ae30-86115db5f6df/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.4118699169921456, + "POST", + "POST /api/v1/chats/a2185e1c-f008-4190-ae30-86115db5f6df/messages: connection/TLS failure (TimeoutError)" + ], + [ + 3.4863260830024956, + "GET", + "GET /api/v1/chats/a2185e1c-f008-4190-ae30-86115db5f6df/messages: connection/TLS failure (TimeoutError)" + ] + ], + "action": { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:10.013737549Z", + "running": false, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + } + } + ], + "rolling_update": { + "label": "sequential API replacement", + "duration_seconds": 24.869, + "successful_requests": 2254, + "successful_writes": 1128, + "errors": 8, + "successful_rps": 90.63, + "first_success_after_action_seconds": { + "POST": 0.01, + "GET": 0.021 + }, + "maximum_success_gap_seconds": { + "POST": 1.04, + "GET": 1.04 + }, + "action_start_seconds": 1.009, + "action_end_seconds": 21.862, + "p50_ms": 8.907, + "p95_ms": 16.321, + "confirmed_data_preserved": true, + "duplicate_messages": 0, + "instances": [ + "19a474c69210", + "63549c0d63ba", + "ca1f62dfc015", + "ed4f3c64a9f6" + ], + "per_second": [ + { + "second": 0, + "successes": 112, + "errors": 0 + }, + { + "second": 1, + "successes": 105, + "errors": 0 + }, + { + "second": 2, + "successes": 104, + "errors": 0 + }, + { + "second": 3, + "successes": 101, + "errors": 0 + }, + { + "second": 4, + "successes": 112, + "errors": 0 + }, + { + "second": 5, + "successes": 46, + "errors": 0 + }, + { + "second": 6, + "successes": 44, + "errors": 4 + }, + { + "second": 7, + "successes": 12, + "errors": 0 + }, + { + "second": 8, + "successes": 112, + "errors": 0 + }, + { + "second": 9, + "successes": 110, + "errors": 0 + }, + { + "second": 10, + "successes": 105, + "errors": 0 + }, + { + "second": 11, + "successes": 111, + "errors": 0 + }, + { + "second": 12, + "successes": 107, + "errors": 0 + }, + { + "second": 13, + "successes": 109, + "errors": 0 + }, + { + "second": 14, + "successes": 51, + "errors": 0 + }, + { + "second": 15, + "successes": 43, + "errors": 4 + }, + { + "second": 16, + "successes": 12, + "errors": 0 + }, + { + "second": 17, + "successes": 107, + "errors": 0 + }, + { + "second": 18, + "successes": 101, + "errors": 0 + }, + { + "second": 19, + "successes": 112, + "errors": 0 + }, + { + "second": 20, + "successes": 112, + "errors": 0 + }, + { + "second": 21, + "successes": 106, + "errors": 0 + }, + { + "second": 22, + "successes": 112, + "errors": 0 + }, + { + "second": 23, + "successes": 111, + "errors": 0 + }, + { + "second": 24, + "successes": 97, + "errors": 0 + } + ], + "error_samples": [ + [ + 6.444048083998496, + "POST", + "POST /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [201], received 503" + ], + [ + 6.461145833993214, + "POST", + "POST /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [201], received 503" + ], + [ + 6.465980209002737, + "GET", + "GET /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [200], received 503" + ], + [ + 6.52059274999192, + "GET", + "GET /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [200], received 503" + ], + [ + 15.429505041989614, + "POST", + "POST /api/v1/chats/0fb92954-72c5-4454-bdb6-48e8b0f30ba9/messages: expected [201], received 503" + ] + ], + "action": { + "before": [ + { + "id": "19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75", + "instance": "19a474c69210", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:33:13.735740384Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0", + "instance": "ca1f62dfc015", + "service": "api", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:33:38.250478257Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", + "service": "cleanup", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:17.411326011Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-11-06T07:31:09.542736424Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "57722" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-11-06T07:32:04.849361089Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab4-check-5aad8e0065_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", + "service": "s3", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:32:11.905651425Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" + } + ] + }, + { + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", + "service": "s3-storage", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:31:17.386629011Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_s3_data" + } + ] + }, + { + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-11-06T07:31:58.693966586Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_valkey_data" + } + ] + } + ], + "after": [ + { + "id": "ed4f3c64a9f6b1e8c80f4e061c2017aae179b3feb768d662fd087f2cc96c52fd", + "instance": "ed4f3c64a9f6", + "service": "api", + "image": "sha256:93c838d55d79a0001346ecab28f72621d9540ba5c52eaeb1ca6020286dd05fd6", + "started_at": "2026-11-06T07:33:50.659312471Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "63549c0d63ba2fcee39baf26eb9ffa0da68e38d2d781a81e4a29f2eff6a0afb8", + "instance": "63549c0d63ba", + "service": "api", + "image": "sha256:93c838d55d79a0001346ecab28f72621d9540ba5c52eaeb1ca6020286dd05fd6", + "started_at": "2026-11-06T07:33:59.668146169Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + } + ] + }, + { + "id": "bfaeabc178285f04b23c0578b9c2d22b8e54ab97c4cb0d70eb874c46dd92acdb", + "instance": "bfaeabc17828", + "service": "cleanup", + "image": "sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d", + "started_at": "2026-11-06T07:31:17.411326011Z", + "running": true, + "exit_code": 0, + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/database_url", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "type": "bind", + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "3bc52373c3d129bac2f4765eaf221fbe4461ae859616920aba3f3cf952a9e449", + "instance": "3bc52373c3d1", + "service": "lb", + "image": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "started_at": "2026-11-06T07:31:09.542736424Z", + "running": true, + "exit_code": 0, + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "57722" + } + ] + }, + "mounts": [ + { + "type": "bind", + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "60274b0d5ab994316a8b828669ce937b08e758c5ce6dfc6d8d915d5cf4c46046", + "instance": "60274b0d5ab9", + "service": "postgres", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "started_at": "2026-11-06T07:32:04.849361089Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/var/lib/postgresql/data", + "name": "lab4-check-5aad8e0065_postgres_data" + }, + { + "type": "bind", + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "0ba4ea69ac8dca26d0348fb351aaf286164ee5e72b2123ffdb89579469af53c9", + "instance": "0ba4ea69ac8d", + "service": "s3", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:32:11.905651425Z", + "running": true, + "exit_code": 0, + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "bind", + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "type": "volume", + "destination": "/data", + "name": "56e17d6e3f080fb7912c461068d6c41768244d709e8a55849e355a0895694340" + } + ] + }, + { + "id": "df540b4c43cfa85275ff158c300583671dbb3eaf12e75b9f7497f5414b3cb36b", + "instance": "df540b4c43cf", + "service": "s3-storage", + "image": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "started_at": "2026-11-06T07:31:17.386629011Z", + "running": true, + "exit_code": 0, + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_s3_data" + } + ] + }, + { + "id": "02dcfb1c815e68cbe7979e537c8d763076b7b59229c03dbfa850124eeef28a12", + "instance": "02dcfb1c815e", + "service": "valkey", + "image": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "started_at": "2026-11-06T07:31:58.693966586Z", + "running": true, + "exit_code": 0, + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": {}, + "mounts": [ + { + "type": "volume", + "destination": "/data", + "name": "lab4-check-5aad8e0065_valkey_data" + } + ] + } + ] + } + }, + "passed": true +} diff --git a/evidence/lab4-checks.txt b/evidence/lab4-checks.txt new file mode 100644 index 0000000..991fcd1 --- /dev/null +++ b/evidence/lab4-checks.txt @@ -0,0 +1,533 @@ +Command: ./scripts/check-docker.sh; project: lab4-check-5aad8e0065 +Docker: 29.0.1 aarch64 CPUs=10 RAM=8217448448 +Local .env and secrets are ready; values are not printed +#1 [internal] load local bake definitions +#1 reading from stdin 564B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 733B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 167B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 1.01kB done +#5 DONE 0.0s + +#6 [1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 done +#6 DONE 0.0s + +#7 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#7 CACHED + +#8 [7/8] COPY migrations/ ./migrations/ +#8 CACHED + +#9 [2/8] WORKDIR /app +#9 CACHED + +#10 [6/8] COPY src/ ./src/ +#10 CACHED + +#11 [3/8] COPY requirements.txt . +#11 CACHED + +#12 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#12 CACHED + +#13 [8/8] COPY research/ ./research/ +#13 CACHED + +#14 exporting to image +#14 exporting layers done +#14 exporting manifest sha256:644f5d408505f5982b75f18404860de692ee837b2b2e18f5c92b0b057081b7ec done +#14 exporting config sha256:726011343b32f5b6277348f2f8b38f95b8c31754ae0d21336375124a18aa2e0e done +#14 exporting attestation manifest sha256:1146b210a20471d58663fb48a3a24ba22986971d0cce8777f912886768f43306 done +#14 exporting manifest list sha256:a04b1bfe2aea8391c8f92dd952499559d766cc4b34091963fe57bcbaf6ec015d done +#14 naming to docker.io/library/mtusi-messenger:lab4 done +#14 unpacking to docker.io/library/mtusi-messenger:lab4 done +#14 DONE 0.1s + +#15 resolving provenance for metadata file +#15 DONE 0.0s + mtusi-messenger:lab4 Built + Network lab4-check-5aad8e0065_default Creating + Network lab4-check-5aad8e0065_default Created + Network lab4-check-5aad8e0065_s3-internal Creating + Network lab4-check-5aad8e0065_s3-internal Created + Volume lab4-check-5aad8e0065_valkey_data Creating + Volume lab4-check-5aad8e0065_valkey_data Created + Volume lab4-check-5aad8e0065_postgres_data Creating + Volume lab4-check-5aad8e0065_postgres_data Created + Volume lab4-check-5aad8e0065_s3_data Creating + Volume lab4-check-5aad8e0065_s3_data Created + Container lab4-check-5aad8e0065-s3-storage-1 Creating + Container lab4-check-5aad8e0065-valkey-1 Creating + Container lab4-check-5aad8e0065-postgres-1 Creating + Container lab4-check-5aad8e0065-valkey-1 Created + Container lab4-check-5aad8e0065-s3-storage-1 Created + Container lab4-check-5aad8e0065-s3-1 Creating + Container lab4-check-5aad8e0065-postgres-1 Created + Container lab4-check-5aad8e0065-migrate-1 Creating + Container lab4-check-5aad8e0065-s3-1 Created + Container lab4-check-5aad8e0065-s3-bootstrap-1 Creating + Container lab4-check-5aad8e0065-migrate-1 Created + Container lab4-check-5aad8e0065-s3-bootstrap-1 Created + Container lab4-check-5aad8e0065-api-2 Creating + Container lab4-check-5aad8e0065-api-1 Creating + Container lab4-check-5aad8e0065-cleanup-1 Creating + Container lab4-check-5aad8e0065-api-1 Created + Container lab4-check-5aad8e0065-cleanup-1 Created + Container lab4-check-5aad8e0065-api-2 Created + Container lab4-check-5aad8e0065-lb-1 Creating + Container lab4-check-5aad8e0065-lb-1 Created + Container lab4-check-5aad8e0065-postgres-1 Starting + Container lab4-check-5aad8e0065-valkey-1 Starting + Container lab4-check-5aad8e0065-s3-storage-1 Starting + Container lab4-check-5aad8e0065-s3-storage-1 Started + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Started + Container lab4-check-5aad8e0065-postgres-1 Started + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Starting + Container lab4-check-5aad8e0065-s3-1 Started + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-2 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-cleanup-1 Starting + Container lab4-check-5aad8e0065-cleanup-1 Started + Container lab4-check-5aad8e0065-api-2 Started + Container lab4-check-5aad8e0065-api-1 Starting + Container lab4-check-5aad8e0065-api-1 Started + Container lab4-check-5aad8e0065-api-2 Waiting + Container lab4-check-5aad8e0065-api-1 Waiting + Container lab4-check-5aad8e0065-api-2 Healthy + Container lab4-check-5aad8e0065-api-1 Healthy + Container lab4-check-5aad8e0065-lb-1 Starting + Container lab4-check-5aad8e0065-lb-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +PASS: 2 real API containers, CPU/RAM limits, API non-root and no public ports; only LB published +Проверяем сервер: http://127.0.0.1:57722 +/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:57722" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +test_chat_access (test_api.ApiTests.test_chat_access) ... ok +test_chat_pagination_and_cursor_owner (test_api.ApiTests.test_chat_pagination_and_cursor_owner) ... ok +test_cursor_cannot_be_used_for_another_chat_or_changed (test_api.ApiTests.test_cursor_cannot_be_used_for_another_chat_or_changed) ... ok +test_default_page_size (test_api.ApiTests.test_default_page_size) ... ok +test_duplicate_username (test_api.ApiTests.test_duplicate_username) ... ok +test_framework_errors_and_request_ids (test_api.ApiTests.test_framework_errors_and_request_ids) ... ok +test_identification_and_authentication_order (test_api.ApiTests.test_identification_and_authentication_order) ... ok +test_invalid_chat_members (test_api.ApiTests.test_invalid_chat_members) ... ok +test_invalid_chat_title (test_api.ApiTests.test_invalid_chat_title) ... ok +test_invalid_limits_and_cursors (test_api.ApiTests.test_invalid_limits_and_cursors) ... ok +test_invalid_message_fields (test_api.ApiTests.test_invalid_message_fields) ... ok +test_invalid_user_fields (test_api.ApiTests.test_invalid_user_fields) ... ok +test_malformed_json_and_encoding (test_api.ApiTests.test_malformed_json_and_encoding) ... ok +test_message_pagination_and_changing_limit (test_api.ApiTests.test_message_pagination_and_changing_limit) ... ok +test_unicode_and_text_preservation (test_api.ApiTests.test_unicode_and_text_preservation) ... ok +test_wrong_content_type (test_api.ApiTests.test_wrong_content_type) ... ok +test_access_and_authentication (test_attachments.AttachmentTests.test_access_and_authentication) ... ok +test_attachment_only_message_and_other_member (test_attachments.AttachmentTests.test_attachment_only_message_and_other_member) ... ok +test_chunked_limit_without_content_length (test_attachments.AttachmentTests.test_chunked_limit_without_content_length) ... ok +test_content_length_limit_before_body (test_attachments.AttachmentTests.test_content_length_limit_before_body) ... ok +test_corruption_and_mime_spoof (test_attachments.AttachmentTests.test_corruption_and_mime_spoof) ... ok +test_filename_rejected_before_storage (test_attachments.AttachmentTests.test_filename_rejected_before_storage) ... ok +test_foreign_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_foreign_and_unknown_attachment_ids) ... ok +test_idempotency_includes_ordered_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_ordered_attachment_ids) ... ok +test_invalid_attachment_lists (test_attachments.AttachmentTests.test_invalid_attachment_lists) ... ok +test_pixel_limit_before_decode (test_attachments.AttachmentTests.test_pixel_limit_before_decode) ... ok +test_png_and_jpeg_originals (test_attachments.AttachmentTests.test_png_and_jpeg_originals) ... ok +test_simultaneous_registration_of_one_username (test_concurrency.ConcurrencyTests.test_simultaneous_registration_of_one_username) ... ok +test_ten_clients_send_one_hundred_messages (test_concurrency.ConcurrencyTests.test_ten_clients_send_one_hundred_messages) ... ok +test_conflict_and_semantic_json_replay (test_idempotency.IdempotencyTests.test_conflict_and_semantic_json_replay) ... ok +test_key_format_and_missing_key (test_idempotency.IdempotencyTests.test_key_format_and_missing_key) ... ok +test_parallel_replay_across_replicas (test_idempotency.IdempotencyTests.test_parallel_replay_across_replicas) ... ok +test_replay_rechecks_session_and_access (test_idempotency.IdempotencyTests.test_replay_rechecks_session_and_access) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_logout_revokes_only_one_session (test_sessions.SessionTests.test_logout_revokes_only_one_session) ... ok +test_password_validation (test_sessions.SessionTests.test_password_validation) ... ok +test_sql_like_message_is_plain_text (test_sessions.SessionTests.test_sql_like_message_is_plain_text) ... ok +test_unicode_password_with_colon_and_private_response (test_sessions.SessionTests.test_unicode_password_with_colon_and_private_response) ... ok +test_wrong_password_and_basic_only_for_login (test_sessions.SessionTests.test_wrong_password_and_basic_only_for_login) ... ok + +---------------------------------------------------------------------- +Ran 39 tests in 9.593s + +OK +PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages. + Container lab4-check-5aad8e0065-postgres-1 Running + Container lab4-check-5aad8e0065-postgres-1 Running +PASS: migrations only in one-shot service; repeated execution preserves data + Container lab4-check-5aad8e0065-cleanup-1 Stopping + Container lab4-check-5aad8e0065-cleanup-1 Stopped + Container lab4-check-5aad8e0065-s3-storage-1 Running + Container lab4-check-5aad8e0065-s3-1 Running + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Running + Container lab4-check-5aad8e0065-s3-1 Running + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy +PASS: private S3, limited credentials, isolated administration, repeated bootstrap +Stopped API: ca1f62dfc015 container=ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0 +API ready through LB: 19a474c69210 + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-2 Starting + Container lab4-check-5aad8e0065-api-2 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +PASS: original downloaded through another replica after stopping uploader +PASS: post-PUT DB failure recorded; cleanup protects fresh, active and ready uploads + Container lab4-check-5aad8e0065-s3-1 Stopping + Container lab4-check-5aad8e0065-s3-1 Stopped + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Starting + Container lab4-check-5aad8e0065-s3-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +PASS: S3 outage gives API 503; recovery preserves original and session + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-cleanup-1 Starting + Container lab4-check-5aad8e0065-cleanup-1 Started +PASS: 10 concurrent real PNG uploads; all checksums match; memory peaks: [215191552, 103378944] + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited +/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:57722" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: both API, LB, PostgreSQL, Valkey, S3 and cleaner actually restarted; data and session preserved +PASS: SIGTERM with unfinished request: {"instance": "19a474c69210", "unfinished_request": true, "shutdown_seconds": 5.523, "new_connections_refused_seconds": 0.185, "exit_code": 0, "oom_killed": false} + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-1 Starting + Container lab4-check-5aad8e0065-api-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 + Container lab4-check-5aad8e0065-valkey-1 Stopping + Container lab4-check-5aad8e0065-valkey-1 Stopped +PASS: valkey outage: API live=200, ready=503, LB ready=503 + Container lab4-check-5aad8e0065-valkey-1 Starting + Container lab4-check-5aad8e0065-valkey-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 + Container lab4-check-5aad8e0065-postgres-1 Stopping + Container lab4-check-5aad8e0065-postgres-1 Stopped +PASS: postgres outage: API live=200, ready=503, LB ready=503 + Container lab4-check-5aad8e0065-postgres-1 Starting + Container lab4-check-5aad8e0065-postgres-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 + Container lab4-check-5aad8e0065-s3-1 Stopping + Container lab4-check-5aad8e0065-s3-1 Stopped +PASS: s3 outage: API live=200, ready=503, LB ready=503 + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Starting + Container lab4-check-5aad8e0065-s3-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 + Container lab4-check-5aad8e0065-postgres-1 Paused + Container lab4-check-5aad8e0065-postgres-1 Unpaused +PASS: paused PostgreSQL: SQL and direct readiness bounded; live works; unpause restores API without restart +/Users/vanek/Desktop/VNPlabs/VNPlab4/mtusi2026lab4/.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:57722" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 110, "failed_attempts": 1, "first_recovery_seconds": 2.467} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-1 Starting + Container lab4-check-5aad8e0065-api-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +Stopped API: 19a474c69210 container=19a474c69210a7be1c6631830e8fa7f251cf6dd3890c7407843090383543ee75 +{"label": "stop 19a474c69210", "duration_seconds": 16.012, "successful_requests": 1420, "successful_writes": 709, "errors": 4, "successful_rps": 88.68, "first_success_after_action_seconds": {"POST": 0.051, "GET": 0.004}, "maximum_success_gap_seconds": {"POST": 2.002, "GET": 1.995}, "action_start_seconds": 1.006, "action_end_seconds": 1.77, "p50_ms": 10.62, "p95_ms": 24.67, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["19a474c69210", "ca1f62dfc015"]} + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-1 Starting + Container lab4-check-5aad8e0065-api-1 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +Stopped API: ca1f62dfc015 container=ca1f62dfc0156fc235dbcd2f52b58eec2282e6ba8d3e056de11ca92758c69df0 +{"label": "stop ca1f62dfc015", "duration_seconds": 16.008, "successful_requests": 1466, "successful_writes": 732, "errors": 4, "successful_rps": 91.58, "first_success_after_action_seconds": {"POST": 0.018, "GET": 0.028}, "maximum_success_gap_seconds": {"POST": 2.002, "GET": 1.999}, "action_start_seconds": 1.002, "action_end_seconds": 1.832, "p50_ms": 9.397, "p95_ms": 18.616, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["19a474c69210", "ca1f62dfc015"]} + Container lab4-check-5aad8e0065-s3-storage-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-storage-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Starting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Started + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-s3-bootstrap-1 Starting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Started + Container lab4-check-5aad8e0065-postgres-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Waiting + Container lab4-check-5aad8e0065-migrate-1 Waiting + Container lab4-check-5aad8e0065-s3-1 Waiting + Container lab4-check-5aad8e0065-s3-bootstrap-1 Waiting + Container lab4-check-5aad8e0065-valkey-1 Healthy + Container lab4-check-5aad8e0065-postgres-1 Healthy + Container lab4-check-5aad8e0065-s3-1 Healthy + Container lab4-check-5aad8e0065-migrate-1 Exited + Container lab4-check-5aad8e0065-s3-bootstrap-1 Exited + Container lab4-check-5aad8e0065-api-2 Starting + Container lab4-check-5aad8e0065-api-2 Started +API ready through LB: 19a474c69210, ca1f62dfc015 +Local .env and secrets are ready; values are not printed +#1 [internal] load local bake definitions +#1 reading from stdin 564B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 733B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#3 DONE 0.0s + +#4 [internal] load .dockerignore +#4 transferring context: 167B done +#4 DONE 0.0s + +#5 [internal] load build context +#5 transferring context: 1.01kB done +#5 DONE 0.0s + +#6 [1/8] FROM docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 +#6 resolve docker.io/library/python:3.11.15-slim-bookworm@sha256:d29f48a31a8b408ed19272ca1e7b10ebae13b240a27e862d3d4217c528e2e0c3 0.0s done +#6 DONE 0.0s + +#7 [3/8] COPY requirements.txt . +#7 CACHED + +#8 [6/8] COPY src/ ./src/ +#8 CACHED + +#9 [2/8] WORKDIR /app +#9 CACHED + +#10 [7/8] COPY migrations/ ./migrations/ +#10 CACHED + +#11 [5/8] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +#11 CACHED + +#12 [4/8] RUN python -m pip install --no-cache-dir -r requirements.txt && python -m pip check +#12 CACHED + +#13 [8/8] COPY research/ ./research/ +#13 CACHED + +#14 exporting to image +#14 exporting layers done +#14 exporting manifest sha256:644f5d408505f5982b75f18404860de692ee837b2b2e18f5c92b0b057081b7ec done +#14 exporting config sha256:726011343b32f5b6277348f2f8b38f95b8c31754ae0d21336375124a18aa2e0e done +#14 exporting attestation manifest sha256:f4a5559518ea1ef0e3747d6084c8ad8dbc8b16dc41cb00062303d0f895b9a235 0.0s done +#14 exporting manifest list sha256:93c838d55d79a0001346ecab28f72621d9540ba5c52eaeb1ca6020286dd05fd6 done +#14 naming to docker.io/library/mtusi-messenger:lab4 done +#14 unpacking to docker.io/library/mtusi-messenger:lab4 done +#14 DONE 0.1s + +#15 resolving provenance for metadata file +#15 DONE 0.0s + mtusi-messenger:lab4 Built + Container lab4-check-5aad8e0065-postgres-1 Running +Migrations: already applied +API ready through LB: 19a474c69210, ca1f62dfc015 +Replacing API: 19a474c69210 + Container lab4-check-5aad8e0065-api-2 Running + Container lab4-check-5aad8e0065-api-3 Creating + Container lab4-check-5aad8e0065-api-3 Created + Container lab4-check-5aad8e0065-api-3 Starting + Container lab4-check-5aad8e0065-api-3 Started +API ready through LB: ca1f62dfc015, ed4f3c64a9f6 +Replacing API: ca1f62dfc015 + Container lab4-check-5aad8e0065-api-3 Running + Container lab4-check-5aad8e0065-api-4 Creating + Container lab4-check-5aad8e0065-api-4 Created + Container lab4-check-5aad8e0065-api-4 Starting + Container lab4-check-5aad8e0065-api-4 Started +API ready through LB: 63549c0d63ba, ed4f3c64a9f6 +PASS: all API containers replaced sequentially; stores and LB kept running +{"label": "sequential API replacement", "duration_seconds": 24.869, "successful_requests": 2254, "successful_writes": 1128, "errors": 8, "successful_rps": 90.63, "first_success_after_action_seconds": {"POST": 0.01, "GET": 0.021}, "maximum_success_gap_seconds": {"POST": 1.04, "GET": 1.04}, "action_start_seconds": 1.009, "action_end_seconds": 21.862, "p50_ms": 8.907, "p95_ms": 16.321, "confirmed_data_preserved": true, "duplicate_messages": 0, "instances": ["19a474c69210", "63549c0d63ba", "ca1f62dfc015", "ed4f3c64a9f6"]} +PASS: all lab4 grade-4 checks completed + Container lab4-check-5aad8e0065-lb-1 Stopping + Container lab4-check-5aad8e0065-cleanup-1 Stopping + Container lab4-check-5aad8e0065-lb-1 Stopped + Container lab4-check-5aad8e0065-lb-1 Removing + Container lab4-check-5aad8e0065-lb-1 Removed + Container lab4-check-5aad8e0065-api-3 Stopping + Container lab4-check-5aad8e0065-api-4 Stopping + Container lab4-check-5aad8e0065-api-3 Stopped + Container lab4-check-5aad8e0065-api-3 Removing + Container lab4-check-5aad8e0065-api-3 Removed + Container lab4-check-5aad8e0065-api-4 Stopped + Container lab4-check-5aad8e0065-api-4 Removing + Container lab4-check-5aad8e0065-api-4 Removed + Container lab4-check-5aad8e0065-valkey-1 Stopping + Container lab4-check-5aad8e0065-valkey-1 Stopped + Container lab4-check-5aad8e0065-valkey-1 Removing + Container lab4-check-5aad8e0065-valkey-1 Removed + Container lab4-check-5aad8e0065-cleanup-1 Stopped + Container lab4-check-5aad8e0065-cleanup-1 Removing + Container lab4-check-5aad8e0065-cleanup-1 Removed + Container lab4-check-5aad8e0065-s3-bootstrap-1 Stopping + Container lab4-check-5aad8e0065-migrate-1 Stopping + Container lab4-check-5aad8e0065-migrate-1 Stopped + Container lab4-check-5aad8e0065-migrate-1 Removing + Container lab4-check-5aad8e0065-s3-bootstrap-1 Stopped + Container lab4-check-5aad8e0065-s3-bootstrap-1 Removing + Container lab4-check-5aad8e0065-s3-bootstrap-1 Removed + Container lab4-check-5aad8e0065-s3-1 Stopping + Container lab4-check-5aad8e0065-migrate-1 Removed + Container lab4-check-5aad8e0065-postgres-1 Stopping + Container lab4-check-5aad8e0065-postgres-1 Stopped + Container lab4-check-5aad8e0065-postgres-1 Removing + Container lab4-check-5aad8e0065-postgres-1 Removed + Container lab4-check-5aad8e0065-s3-1 Stopped + Container lab4-check-5aad8e0065-s3-1 Removing + Container lab4-check-5aad8e0065-s3-1 Removed + Container lab4-check-5aad8e0065-s3-storage-1 Stopping + Container lab4-check-5aad8e0065-s3-storage-1 Stopped + Container lab4-check-5aad8e0065-s3-storage-1 Removing + Container lab4-check-5aad8e0065-s3-storage-1 Removed + Volume lab4-check-5aad8e0065_valkey_data Removing + Network lab4-check-5aad8e0065_s3-internal Removing + Volume lab4-check-5aad8e0065_postgres_data Removing + Volume lab4-check-5aad8e0065_s3_data Removing + Network lab4-check-5aad8e0065_default Removing + Volume lab4-check-5aad8e0065_valkey_data Removed + Volume lab4-check-5aad8e0065_s3_data Removed + Volume lab4-check-5aad8e0065_postgres_data Removed + Network lab4-check-5aad8e0065_s3-internal Removed + Network lab4-check-5aad8e0065_default Removed +Removed only temporary test containers and volumes: lab4-check-5aad8e0065 diff --git a/evidence/lab5-checks.json b/evidence/lab5-checks.json new file mode 100644 index 0000000..1acd060 --- /dev/null +++ b/evidence/lab5-checks.json @@ -0,0 +1,502 @@ +{ + "utc": "2026-11-12T15:23:27.573351+00:00", + "command": "./scripts/check-docker.sh", + "project": "lab5-check-d233766d34", + "host": "Darwin arm64", + "source_commit": "276dbdad8a34da4c5b57fbf1879ca169b581d75b", + "working_tree": true, + "configuration": { + "concurrency": 2, + "prefetch": 1, + "attempts": 5, + "timeout_seconds": 5, + "lease_seconds": 15, + "backoff_base_seconds": 0.15 + }, + "checks": { + "topology_and_limits": { + "passed": true, + "api_replicas": 2, + "broker": "Valkey Streams with AOF fsync always", + "worker_processes": 2 + }, + "public_smoke_and_schema": { + "passed": true + }, + "student_api_regressions": { + "passed": true + }, + "worker_offline_api_broker_restart": { + "passed": true, + "attachment_id": "63406ab0-cc3c-4fa2-a0aa-e5e3353d5ae3", + "job_id": "7f450ff7-cb2d-4189-a668-f1967dbd94e8", + "message_id": "bc37bafb-efe1-4481-b304-c725c6e8368d", + "variants": { + "thumbnail": { + "size": [ + 256, + 160 + ], + "bytes": 5813 + }, + "crop": { + "size": [ + 128, + 128 + ], + "bytes": 2462 + }, + "optimized": { + "size": [ + 320, + 200 + ], + "bytes": 2462 + } + } + }, + "outbox_crash_before_publish": { + "passed": true, + "job_id": "a3973e2b-ff73-4bca-bc41-394f2b875d13", + "automatic_recovery": true + }, + "outbox_crash_after_publish": { + "passed": true, + "job_id": "007cb1d1-4b3b-4d12-9b2b-91a4682e8b08", + "automatic_recovery": true + }, + "worker_crash_after_objects": { + "passed": true, + "job_id": "21129707-7dd0-4a72-ac33-5691dae3194e", + "attempt": 2, + "canonical_objects": 4, + "duplicate_noop": true + }, + "worker_crash_before_ack": { + "passed": true, + "job_id": "2eeba68e-3fc7-42e0-adf9-13ac01c6c982", + "attempt": 1, + "canonical_objects": 4, + "duplicate_noop": true + }, + "bounded_retry_failed_registry_replay": { + "passed": true, + "job_id": "bf756bf3-c3ca-45d2-acdd-3e67eb882d4d", + "max_automatic_attempts": 5, + "audit": [ + { + "event": "processing", + "attempt": 1, + "error_code": null, + "at": "2026-11-12T15:25:18.431558+00:00" + }, + { + "event": "retry", + "attempt": 1, + "error_code": "storage_unavailable", + "at": "2026-11-12T15:25:18.432821+00:00" + }, + { + "event": "processing", + "attempt": 2, + "error_code": null, + "at": "2026-11-12T15:25:18.692895+00:00" + }, + { + "event": "retry", + "attempt": 2, + "error_code": "storage_unavailable", + "at": "2026-11-12T15:25:18.69618+00:00" + }, + { + "event": "processing", + "attempt": 3, + "error_code": null, + "at": "2026-11-12T15:25:19.206061+00:00" + }, + { + "event": "retry", + "attempt": 3, + "error_code": "storage_unavailable", + "at": "2026-11-12T15:25:19.208065+00:00" + }, + { + "event": "processing", + "attempt": 4, + "error_code": null, + "at": "2026-11-12T15:25:19.97778+00:00" + }, + { + "event": "retry", + "attempt": 4, + "error_code": "storage_unavailable", + "at": "2026-11-12T15:25:19.978984+00:00" + }, + { + "event": "processing", + "attempt": 5, + "error_code": null, + "at": "2026-11-12T15:25:21.777757+00:00" + }, + { + "event": "failed", + "attempt": 5, + "error_code": "retry_exhausted", + "at": "2026-11-12T15:25:21.78129+00:00" + }, + { + "event": "operator_replay", + "attempt": 5, + "error_code": null, + "at": "2026-11-12T15:25:22.680791+00:00" + }, + { + "event": "processing", + "attempt": 6, + "error_code": null, + "at": "2026-11-12T15:25:22.802027+00:00" + }, + { + "event": "ready", + "attempt": 6, + "error_code": null, + "at": "2026-11-12T15:25:23.151043+00:00" + } + ], + "ready_replay_rejected": true + }, + "permanent_error": { + "passed": true, + "job_id": "c2056d32-08bf-4c8a-8980-2f8b6c2511c9", + "attempt": 1, + "error_code": "invalid_image" + }, + "image_subprocess_timeout": { + "passed": true, + "job_id": "6e8efff5-d82d-45f4-af12-3b05dcdb0076", + "timeout_seconds": 5, + "task_memory_mb": 640, + "recovered": true + }, + "image_subprocess_memory": { + "passed": true, + "job_id": "ff1e7e76-b82d-4ba1-8ca9-79f3d78a4dfe", + "timeout_seconds": 5, + "task_memory_mb": 640, + "recovered": true + }, + "image_semantics": { + "passed": true, + "cases": [ + { + "case": "small_alpha_on_white", + "variants": { + "thumbnail": { + "size": [ + 17, + 9 + ], + "bytes": 635 + }, + "crop": { + "size": [ + 128, + 128 + ], + "bytes": 883 + }, + "optimized": { + "size": [ + 17, + 9 + ], + "bytes": 56 + } + } + }, + { + "case": "exif_orientation_6_and_floor", + "variants": { + "thumbnail": { + "size": [ + 160, + 256 + ], + "bytes": 1270 + }, + "crop": { + "size": [ + 128, + 128 + ], + "bytes": 886 + }, + "optimized": { + "size": [ + 201, + 321 + ], + "bytes": 210 + } + } + } + ] + }, + "staging_cleanup": { + "passed": true, + "active_preserved": true, + "orphan_removed": true, + "ready_removed": true, + "expired_failed_removed": true, + "expired_replay_rejected": true + }, + "broker_offline_outbox": { + "passed": true, + "attachment_id": "3b3d3600-efcc-4840-ab76-07868501c21d", + "recovered": true + }, + "public_persistence": { + "passed": true + }, + "public_api_failover": { + "passed": true + } + }, + "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", + "containers": [ + { + "id": "9fbe552bf1bd8c2be76b3434cc5662479ca8d328ca698813ac9a785bcafc0cfd", + "service": "api", + "image": "mtusi-messenger:lab5", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "destination": "/run/secrets/database_url", + "name": null + }, + { + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "destination": "/run/secrets/s3_access_key_id", + "name": null + } + ] + }, + { + "id": "ebc82ed28847cbaf6f474088c68005f05d49469010b45ff3767fb0a944ead6e0", + "service": "api", + "image": "mtusi-messenger:lab5", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "destination": "/run/secrets/database_url", + "name": null + }, + { + "destination": "/run/secrets/cache_url", + "name": null + } + ] + }, + { + "id": "26a48bd7b3390a4cd99667d4d99d6342edce8e26accaee3b181a4d842203261a", + "service": "broker", + "image": "valkey/valkey:8.1.3", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/data", + "name": "lab5-check-d233766d34_broker_data" + }, + { + "destination": "/run/secrets/broker_password", + "name": null + } + ] + }, + { + "id": "232fdc281b939bf905b4123102a6a6eb609a4ffcb730f6d03707beb07283d69c", + "service": "cleanup", + "image": "mtusi-messenger:lab5", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/database_url", + "name": null + }, + { + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "e722801286bb112062f1da59af65dc21eddf5059ca086022e0d52f85a0f26154", + "service": "lb", + "image": "haproxy:3.2.6-alpine", + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "ba806a4ce014a516ea867a516743de64bef0d449726bb544d265bf31b4055b1b", + "service": "migrate", + "image": "mtusi-messenger:lab5", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "0fd62d960020290ba4089d05fdcc8da2bb0e2faf7528fc6ed3384e35236d4290", + "service": "postgres", + "image": "postgres:17.6-bookworm", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/var/lib/postgresql/data", + "name": "lab5-check-d233766d34_postgres_data" + }, + { + "destination": "/run/secrets/postgres_password", + "name": null + } + ] + }, + { + "id": "c658ec3ef1e14ca2f1ca8f1530e6f13b37b99ce8389fafeae84e841e65daa2fc", + "service": "publisher", + "image": "mtusi-messenger:lab5", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/database_url", + "name": null + }, + { + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "id": "bc944a3a03374052aba43f42767c253755a921090fd15ead92fa2a924507c941", + "service": "s3", + "image": "chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "destination": "/data", + "name": "0496c4617fd5cac2843b4c391fc1eb42e15c90c4d9a35d092e71a6fd8815960f" + } + ] + }, + { + "id": "9e7382c34331cb65974fa3feb248105c8055fa25c251151f0f004b65fc388923", + "service": "s3-bootstrap", + "image": "mtusi-messenger:lab5", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/run/secrets/s3_root_password", + "name": null + }, + { + "destination": "/run/secrets/s3_root_user", + "name": null + } + ] + }, + { + "id": "b9821d3ec48ce18b8255a446cf99b02d6ea9658de406a15cc0ace7ccf7d7350e", + "service": "s3-storage", + "image": "chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/data", + "name": "lab5-check-d233766d34_s3_data" + } + ] + }, + { + "id": "c8ddc9fede2be42d05e79147dfc7dd87efede3b67f3226afeee1161fcaa49428", + "service": "valkey", + "image": "valkey/valkey:8.1.3", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/data", + "name": "lab5-check-d233766d34_valkey_data" + } + ] + }, + { + "id": "9fee2e7832c5c11e81bb56df9d435ad090fdb1fddb85efbf014ea6019559ed75", + "service": "worker", + "image": "mtusi-messenger:lab5", + "memory_bytes": 1610612736, + "nano_cpus": 2000000000, + "mounts": [ + { + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "destination": "/run/secrets/database_url", + "name": null + } + ] + } + ], + "passed": true +} diff --git a/evidence/lab5-checks.txt b/evidence/lab5-checks.txt new file mode 100644 index 0000000..61c1241 --- /dev/null +++ b/evidence/lab5-checks.txt @@ -0,0 +1,77 @@ +PASS: topology_and_limits {"api_replicas": 2, "broker": "Valkey Streams with AOF fsync always", "worker_processes": 2} +/Users/vanek/Desktop/VNPlabs/VNPlab5/mtusi2026lab5/.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +/Users/vanek/Desktop/VNPlabs/VNPlab5/mtusi2026lab5/.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +/Users/vanek/Desktop/VNPlabs/VNPlab5/mtusi2026lab5/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:57277" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +PASS: public_smoke_and_schema {} +PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages. +test_chat_access (test_api.ApiTests.test_chat_access) ... ok +test_chat_pagination_and_cursor_owner (test_api.ApiTests.test_chat_pagination_and_cursor_owner) ... ok +test_cursor_cannot_be_used_for_another_chat_or_changed (test_api.ApiTests.test_cursor_cannot_be_used_for_another_chat_or_changed) ... ok +test_default_page_size (test_api.ApiTests.test_default_page_size) ... ok +test_duplicate_username (test_api.ApiTests.test_duplicate_username) ... ok +test_framework_errors_and_request_ids (test_api.ApiTests.test_framework_errors_and_request_ids) ... ok +test_identification_and_authentication_order (test_api.ApiTests.test_identification_and_authentication_order) ... ok +test_invalid_chat_members (test_api.ApiTests.test_invalid_chat_members) ... ok +test_invalid_chat_title (test_api.ApiTests.test_invalid_chat_title) ... ok +test_invalid_limits_and_cursors (test_api.ApiTests.test_invalid_limits_and_cursors) ... ok +test_invalid_message_fields (test_api.ApiTests.test_invalid_message_fields) ... ok +test_invalid_user_fields (test_api.ApiTests.test_invalid_user_fields) ... ok +test_malformed_json_and_encoding (test_api.ApiTests.test_malformed_json_and_encoding) ... ok +test_message_pagination_and_changing_limit (test_api.ApiTests.test_message_pagination_and_changing_limit) ... ok +test_unicode_and_text_preservation (test_api.ApiTests.test_unicode_and_text_preservation) ... ok +test_wrong_content_type (test_api.ApiTests.test_wrong_content_type) ... ok +test_access_and_authentication (test_attachments.AttachmentTests.test_access_and_authentication) ... ok +test_attachment_only_message_and_other_member (test_attachments.AttachmentTests.test_attachment_only_message_and_other_member) ... ok +test_chunked_limit_without_content_length (test_attachments.AttachmentTests.test_chunked_limit_without_content_length) ... ok +test_content_length_limit_before_body (test_attachments.AttachmentTests.test_content_length_limit_before_body) ... ok +test_corruption_and_mime_spoof (test_attachments.AttachmentTests.test_corruption_and_mime_spoof) ... ok +test_filename_rejected_before_storage (test_attachments.AttachmentTests.test_filename_rejected_before_storage) ... ok +test_foreign_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_foreign_and_unknown_attachment_ids) ... ok +test_idempotency_includes_ordered_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_ordered_attachment_ids) ... ok +test_invalid_attachment_lists (test_attachments.AttachmentTests.test_invalid_attachment_lists) ... ok +test_pixel_limit_before_decode (test_attachments.AttachmentTests.test_pixel_limit_before_decode) ... ok +test_png_and_jpeg_originals (test_attachments.AttachmentTests.test_png_and_jpeg_originals) ... ok +test_simultaneous_registration_of_one_username (test_concurrency.ConcurrencyTests.test_simultaneous_registration_of_one_username) ... ok +test_ten_clients_send_one_hundred_messages (test_concurrency.ConcurrencyTests.test_ten_clients_send_one_hundred_messages) ... ok +test_conflict_and_semantic_json_replay (test_idempotency.IdempotencyTests.test_conflict_and_semantic_json_replay) ... ok +test_key_format_and_missing_key (test_idempotency.IdempotencyTests.test_key_format_and_missing_key) ... ok +test_parallel_replay_across_replicas (test_idempotency.IdempotencyTests.test_parallel_replay_across_replicas) ... ok +test_replay_rechecks_session_and_access (test_idempotency.IdempotencyTests.test_replay_rechecks_session_and_access) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_logout_revokes_only_one_session (test_sessions.SessionTests.test_logout_revokes_only_one_session) ... ok +test_password_validation (test_sessions.SessionTests.test_password_validation) ... ok +test_sql_like_message_is_plain_text (test_sessions.SessionTests.test_sql_like_message_is_plain_text) ... ok +test_unicode_password_with_colon_and_private_response (test_sessions.SessionTests.test_unicode_password_with_colon_and_private_response) ... ok +test_wrong_password_and_basic_only_for_login (test_sessions.SessionTests.test_wrong_password_and_basic_only_for_login) ... ok + +---------------------------------------------------------------------- +Ran 39 tests in 6.981s + +OK +PASS: student_api_regressions {} +PASS: worker_offline_api_broker_restart {"attachment_id": "63406ab0-cc3c-4fa2-a0aa-e5e3353d5ae3", "job_id": "7f450ff7-cb2d-4189-a668-f1967dbd94e8", "message_id": "bc37bafb-efe1-4481-b304-c725c6e8368d", "variants": {"thumbnail": {"size": [256, 160], "bytes": 5813}, "crop": {"size": [128, 128], "bytes": 2462}, "optimized": {"size": [320, 200], "bytes": 2462}}} +PASS: outbox_crash_before_publish {"job_id": "a3973e2b-ff73-4bca-bc41-394f2b875d13", "automatic_recovery": true} +PASS: outbox_crash_after_publish {"job_id": "007cb1d1-4b3b-4d12-9b2b-91a4682e8b08", "automatic_recovery": true} +PASS: worker_crash_after_objects {"job_id": "21129707-7dd0-4a72-ac33-5691dae3194e", "attempt": 2, "canonical_objects": 4, "duplicate_noop": true} +PASS: worker_crash_before_ack {"job_id": "2eeba68e-3fc7-42e0-adf9-13ac01c6c982", "attempt": 1, "canonical_objects": 4, "duplicate_noop": true} +PASS: bounded_retry_failed_registry_replay {"job_id": "bf756bf3-c3ca-45d2-acdd-3e67eb882d4d", "max_automatic_attempts": 5, "audit": [{"event": "processing", "attempt": 1, "error_code": null, "at": "2026-11-12T15:25:18.431558+00:00"}, {"event": "retry", "attempt": 1, "error_code": "storage_unavailable", "at": "2026-11-12T15:25:18.432821+00:00"}, {"event": "processing", "attempt": 2, "error_code": null, "at": "2026-11-12T15:25:18.692895+00:00"}, {"event": "retry", "attempt": 2, "error_code": "storage_unavailable", "at": "2026-11-12T15:25:18.69618+00:00"}, {"event": "processing", "attempt": 3, "error_code": null, "at": "2026-11-12T15:25:19.206061+00:00"}, {"event": "retry", "attempt": 3, "error_code": "storage_unavailable", "at": "2026-11-12T15:25:19.208065+00:00"}, {"event": "processing", "attempt": 4, "error_code": null, "at": "2026-11-12T15:25:19.97778+00:00"}, {"event": "retry", "attempt": 4, "error_code": "storage_unavailable", "at": "2026-11-12T15:25:19.978984+00:00"}, {"event": "processing", "attempt": 5, "error_code": null, "at": "2026-11-12T15:25:21.777757+00:00"}, {"event": "failed", "attempt": 5, "error_code": "retry_exhausted", "at": "2026-11-12T15:25:21.78129+00:00"}, {"event": "operator_replay", "attempt": 5, "error_code": null, "at": "2026-11-12T15:25:22.680791+00:00"}, {"event": "processing", "attempt": 6, "error_code": null, "at": "2026-11-12T15:25:22.802027+00:00"}, {"event": "ready", "attempt": 6, "error_code": null, "at": "2026-11-12T15:25:23.151043+00:00"}], "ready_replay_rejected": true} +PASS: permanent_error {"job_id": "c2056d32-08bf-4c8a-8980-2f8b6c2511c9", "attempt": 1, "error_code": "invalid_image"} +PASS: image_subprocess_timeout {"job_id": "6e8efff5-d82d-45f4-af12-3b05dcdb0076", "timeout_seconds": 5, "task_memory_mb": 640, "recovered": true} +PASS: image_subprocess_memory {"job_id": "ff1e7e76-b82d-4ba1-8ca9-79f3d78a4dfe", "timeout_seconds": 5, "task_memory_mb": 640, "recovered": true} +PASS: image_semantics {"cases": [{"case": "small_alpha_on_white", "variants": {"thumbnail": {"size": [17, 9], "bytes": 635}, "crop": {"size": [128, 128], "bytes": 883}, "optimized": {"size": [17, 9], "bytes": 56}}}, {"case": "exif_orientation_6_and_floor", "variants": {"thumbnail": {"size": [160, 256], "bytes": 1270}, "crop": {"size": [128, 128], "bytes": 886}, "optimized": {"size": [201, 321], "bytes": 210}}}]} +PASS: staging_cleanup {"active_preserved": true, "orphan_removed": true, "ready_removed": true, "expired_failed_removed": true, "expired_replay_rejected": true} +PASS: broker_offline_outbox {"attachment_id": "3b3d3600-efcc-4840-ab76-07868501c21d", "recovered": true} +/Users/vanek/Desktop/VNPlabs/VNPlab5/mtusi2026lab5/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:57277" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: public_persistence {} +/Users/vanek/Desktop/VNPlabs/VNPlab5/mtusi2026lab5/.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:57277" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 110, "failed_attempts": 1, "first_recovery_seconds": 2.797} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: public_api_failover {} +PASS: all lab5 grade-4 criteria completed +Removed isolated test stack: lab5-check-d233766d34 diff --git a/evidence/lab5-restart.json b/evidence/lab5-restart.json new file mode 100644 index 0000000..0bca023 --- /dev/null +++ b/evidence/lab5-restart.json @@ -0,0 +1,140 @@ +{ + "utc": "2026-11-12T15:28:31.543303+00:00", + "command": ".venv/bin/python student-tests/verify_restart.py", + "passed": true, + "before": { + "9e5a9d3d2bf0b2e47ee6d85b40b1899881db0d4d91d9ec09e50fb36aca67833c": { + "service": "api", + "started_at": "2026-11-12T15:28:13.153166256Z", + "volumes": [] + }, + "83bab824b05accd2aa8829e83e272d72d4105644f8270cf3dc15f3412d569f5b": { + "service": "api", + "started_at": "2026-11-12T15:28:13.038450506Z", + "volumes": [] + }, + "6f72735cbaed44f82555dcd550199764ab53a8c61a63c88f25c612fd39b6baba": { + "service": "broker", + "started_at": "2026-11-12T15:27:46.698266716Z", + "volumes": [ + "mtusi-lab5_broker_data" + ] + }, + "0f9060c7dc8473fa926c5affc560b3478b90c77771de9810ebba7858a30838d5": { + "service": "cleanup", + "started_at": "2026-11-12T15:28:13.036348173Z", + "volumes": [] + }, + "d381adfb7273a5dd2f22d361a6e1ff490e5c2cf44ff147a8253f7f940cd01de4": { + "service": "lb", + "started_at": "2026-11-12T15:28:18.76757655Z", + "volumes": [] + }, + "9906c3edca610723e6f09b64867fa71a543f18ef4b474804414e9f818a803115": { + "service": "postgres", + "started_at": "2026-11-12T15:27:46.700095216Z", + "volumes": [ + "mtusi-lab5_postgres_data" + ] + }, + "d705dceffd740900bbad04ee520bb96247f286c7c5381734a28bc1c9e30f27e4": { + "service": "publisher", + "started_at": "2026-11-12T15:27:49.924416218Z", + "volumes": [] + }, + "cbe9063ff82960d9db58bc5713af90e4be3e72fdd1c08b0a92a93849dc6ad1aa": { + "service": "s3", + "started_at": "2026-11-12T15:28:09.808290754Z", + "volumes": [ + "fe34bef7724beefe41335cd197f83aae91b16df70766cee6321bff12f765bc66" + ] + }, + "6d1b6c402e19a068a0f2f4b9c06d5abbec7e41b5d778a883e6d045be409f6d4c": { + "service": "s3-storage", + "started_at": "2026-11-12T15:27:46.699445508Z", + "volumes": [ + "mtusi-lab5_s3_data" + ] + }, + "6d8c3e69c4b60acaf42e0259d367a0a3f2c06081a6349c284ddf9f2e9b5ba399": { + "service": "valkey", + "started_at": "2026-11-12T15:27:46.697619716Z", + "volumes": [ + "mtusi-lab5_valkey_data" + ] + }, + "28fabe35a9a5b57e0a3a96bb4ae5308fbd2fa825470799180a2471b55400206e": { + "service": "worker", + "started_at": "2026-11-12T15:28:13.037172673Z", + "volumes": [] + } + }, + "after": { + "9e5a9d3d2bf0b2e47ee6d85b40b1899881db0d4d91d9ec09e50fb36aca67833c": { + "service": "api", + "started_at": "2026-11-12T15:28:33.373854585Z", + "volumes": [] + }, + "83bab824b05accd2aa8829e83e272d72d4105644f8270cf3dc15f3412d569f5b": { + "service": "api", + "started_at": "2026-11-12T15:28:33.338416001Z", + "volumes": [] + }, + "6f72735cbaed44f82555dcd550199764ab53a8c61a63c88f25c612fd39b6baba": { + "service": "broker", + "started_at": "2026-11-12T15:28:33.257433543Z", + "volumes": [ + "mtusi-lab5_broker_data" + ] + }, + "0f9060c7dc8473fa926c5affc560b3478b90c77771de9810ebba7858a30838d5": { + "service": "cleanup", + "started_at": "2026-11-12T15:28:40.990298547Z", + "volumes": [] + }, + "d381adfb7273a5dd2f22d361a6e1ff490e5c2cf44ff147a8253f7f940cd01de4": { + "service": "lb", + "started_at": "2026-11-12T15:28:33.069501668Z", + "volumes": [] + }, + "9906c3edca610723e6f09b64867fa71a543f18ef4b474804414e9f818a803115": { + "service": "postgres", + "started_at": "2026-11-12T15:28:33.134879876Z", + "volumes": [ + "mtusi-lab5_postgres_data" + ] + }, + "d705dceffd740900bbad04ee520bb96247f286c7c5381734a28bc1c9e30f27e4": { + "service": "publisher", + "started_at": "2026-11-12T15:28:40.90857463Z", + "volumes": [] + }, + "cbe9063ff82960d9db58bc5713af90e4be3e72fdd1c08b0a92a93849dc6ad1aa": { + "service": "s3", + "started_at": "2026-11-12T15:28:33.167460626Z", + "volumes": [ + "fe34bef7724beefe41335cd197f83aae91b16df70766cee6321bff12f765bc66" + ] + }, + "6d1b6c402e19a068a0f2f4b9c06d5abbec7e41b5d778a883e6d045be409f6d4c": { + "service": "s3-storage", + "started_at": "2026-11-12T15:28:40.950084172Z", + "volumes": [ + "mtusi-lab5_s3_data" + ] + }, + "6d8c3e69c4b60acaf42e0259d367a0a3f2c06081a6349c284ddf9f2e9b5ba399": { + "service": "valkey", + "started_at": "2026-11-12T15:28:33.205284001Z", + "volumes": [ + "mtusi-lab5_valkey_data" + ] + }, + "28fabe35a9a5b57e0a3a96bb4ae5308fbd2fa825470799180a2471b55400206e": { + "service": "worker", + "started_at": "2026-11-12T15:28:33.299354043Z", + "volumes": [] + } + }, + "same_containers_and_volumes": true +} diff --git a/evidence/lab5-restart.txt b/evidence/lab5-restart.txt new file mode 100644 index 0000000..92f4ca9 --- /dev/null +++ b/evidence/lab5-restart.txt @@ -0,0 +1,4 @@ +/Users/vanek/Desktop/VNPlabs/VNPlab5/mtusi2026lab5/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: all persistent services actually restarted; container IDs and volumes preserved diff --git a/evidence/lab6-api-checks.txt b/evidence/lab6-api-checks.txt new file mode 100644 index 0000000..4f77ec3 --- /dev/null +++ b/evidence/lab6-api-checks.txt @@ -0,0 +1,51 @@ +.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +.venv/bin/python tests/smoke.py --base-url "http://localhost:8080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +test_chat_access (test_api.ApiTests.test_chat_access) ... ok +test_chat_pagination_and_cursor_owner (test_api.ApiTests.test_chat_pagination_and_cursor_owner) ... ok +test_cursor_cannot_be_used_for_another_chat_or_changed (test_api.ApiTests.test_cursor_cannot_be_used_for_another_chat_or_changed) ... ok +test_default_page_size (test_api.ApiTests.test_default_page_size) ... ok +test_duplicate_username (test_api.ApiTests.test_duplicate_username) ... ok +test_framework_errors_and_request_ids (test_api.ApiTests.test_framework_errors_and_request_ids) ... ok +test_identification_and_authentication_order (test_api.ApiTests.test_identification_and_authentication_order) ... ok +test_invalid_chat_members (test_api.ApiTests.test_invalid_chat_members) ... ok +test_invalid_chat_title (test_api.ApiTests.test_invalid_chat_title) ... ok +test_invalid_limits_and_cursors (test_api.ApiTests.test_invalid_limits_and_cursors) ... ok +test_invalid_message_fields (test_api.ApiTests.test_invalid_message_fields) ... ok +test_invalid_user_fields (test_api.ApiTests.test_invalid_user_fields) ... ok +test_malformed_json_and_encoding (test_api.ApiTests.test_malformed_json_and_encoding) ... ok +test_message_pagination_and_changing_limit (test_api.ApiTests.test_message_pagination_and_changing_limit) ... ok +test_unicode_and_text_preservation (test_api.ApiTests.test_unicode_and_text_preservation) ... ok +test_wrong_content_type (test_api.ApiTests.test_wrong_content_type) ... ok +test_access_and_authentication (test_attachments.AttachmentTests.test_access_and_authentication) ... ok +test_attachment_only_message_and_other_member (test_attachments.AttachmentTests.test_attachment_only_message_and_other_member) ... ok +test_chunked_limit_without_content_length (test_attachments.AttachmentTests.test_chunked_limit_without_content_length) ... ok +test_content_length_limit_before_body (test_attachments.AttachmentTests.test_content_length_limit_before_body) ... ok +test_corruption_and_mime_spoof (test_attachments.AttachmentTests.test_corruption_and_mime_spoof) ... ok +test_filename_rejected_before_storage (test_attachments.AttachmentTests.test_filename_rejected_before_storage) ... ok +test_foreign_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_foreign_and_unknown_attachment_ids) ... ok +test_idempotency_includes_ordered_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_ordered_attachment_ids) ... ok +test_invalid_attachment_lists (test_attachments.AttachmentTests.test_invalid_attachment_lists) ... ok +test_pixel_limit_before_decode (test_attachments.AttachmentTests.test_pixel_limit_before_decode) ... ok +test_png_and_jpeg_originals (test_attachments.AttachmentTests.test_png_and_jpeg_originals) ... ok +test_simultaneous_registration_of_one_username (test_concurrency.ConcurrencyTests.test_simultaneous_registration_of_one_username) ... ok +test_ten_clients_send_one_hundred_messages (test_concurrency.ConcurrencyTests.test_ten_clients_send_one_hundred_messages) ... ok +test_conflict_and_semantic_json_replay (test_idempotency.IdempotencyTests.test_conflict_and_semantic_json_replay) ... ok +test_key_format_and_missing_key (test_idempotency.IdempotencyTests.test_key_format_and_missing_key) ... ok +test_parallel_replay_across_replicas (test_idempotency.IdempotencyTests.test_parallel_replay_across_replicas) ... ok +test_replay_rechecks_session_and_access (test_idempotency.IdempotencyTests.test_replay_rechecks_session_and_access) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_logout_revokes_only_one_session (test_sessions.SessionTests.test_logout_revokes_only_one_session) ... ok +test_password_validation (test_sessions.SessionTests.test_password_validation) ... ok +test_sql_like_message_is_plain_text (test_sessions.SessionTests.test_sql_like_message_is_plain_text) ... ok +test_unicode_password_with_colon_and_private_response (test_sessions.SessionTests.test_unicode_password_with_colon_and_private_response) ... ok +test_wrong_password_and_basic_only_for_login (test_sessions.SessionTests.test_wrong_password_and_basic_only_for_login) ... ok + +---------------------------------------------------------------------- +Ran 39 tests in 7.518s + +OK +PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages. diff --git a/evidence/lab6-checks.json b/evidence/lab6-checks.json new file mode 100644 index 0000000..3c9baa7 --- /dev/null +++ b/evidence/lab6-checks.json @@ -0,0 +1,102 @@ +{ + "utc": "2026-11-19T15:09:33.967862+00:00", + "checks": { + "retry_trace_and_logs": { + "passed": true, + "trace_id": "93e1177f9568f5bbfc0b6967a1324a84", + "request_id": "422b23e9-c8f9-4810-ad09-1cb99b6c9576", + "job_id": "fd4d7cce-b0dc-4edd-9938-61230afa7370", + "span_count": 59 + }, + "metrics_and_cardinality": { + "passed": true, + "series": 1192 + }, + "backend_outage": { + "passed": true, + "backend": "loki", + "successful_writes": 24, + "outage_seconds": 51.39, + "missing_request_logs": 6, + "tested_request_ids": [ + "a5835a4b-40b6-4383-90a3-82d07b5bdcfb", + "ee8b61c0-9acc-4ff4-a0fd-f4a7d84e29a2", + "bb492157-8ffd-49a6-84ef-c2b17379bc8e", + "6a076038-13e0-415d-b961-254cb7d48194", + "59ab7cdd-f319-4828-a0b2-4bb9ecef9464", + "0b472396-a1f4-453c-aff2-0a8e2650f3e3", + "b0916ed0-b218-423a-ac39-1c6e904300e6", + "bcd32c58-953b-4759-ac8b-e87fc66c4fd4", + "0a065b55-3fac-46fc-9d92-c67f8ba6e409", + "11d46ead-2941-4279-aa86-0abf111bcaba", + "f6807acd-4e5f-4bef-9f35-93b2ac581390", + "fec97823-5700-4fab-a18a-8a98f11f5c4e", + "a03b0d52-34ea-4b0a-86f2-ac408a0a107c", + "6d44ee0d-f039-406b-b0f7-4ceb7743d020", + "efc0b1d3-e889-4ab4-a169-0369c4cb260b", + "0ae460c2-6b26-4f5c-8959-ecb48a61ca92", + "0f59b3eb-2187-412b-b569-c2373748020e", + "086464d0-5b79-4c9d-aa95-e08af7858c43", + "4c42e246-b973-4e79-8a37-be114b255b80", + "2065d6ba-c0f8-43a5-999f-55ec5299d855", + "8ccb2503-c32f-4b3d-9763-2d9455f13a99", + "979ef391-bc49-42af-95b8-194e028dd796", + "fe3a4f81-114e-4055-b160-7f47a15acfff", + "e6019f30-5b26-4bc1-a630-bcfbbeb6ba96" + ], + "missing_request_ids": [ + "a5835a4b-40b6-4383-90a3-82d07b5bdcfb", + "ee8b61c0-9acc-4ff4-a0fd-f4a7d84e29a2", + "bb492157-8ffd-49a6-84ef-c2b17379bc8e", + "6a076038-13e0-415d-b961-254cb7d48194", + "59ab7cdd-f319-4828-a0b2-4bb9ecef9464", + "0b472396-a1f4-453c-aff2-0a8e2650f3e3" + ], + "collector_memory_samples": [ + "{\"BlockIO\":\"0B / 0B\",\"CPUPerc\":\"0.12%\",\"Container\":\"6e21c63012d160a406e28a342d668de0be55107aa16228ed9d5dc15c34b447be\",\"ID\":\"6e21c63012d1\",\"MemPerc\":\"28.44%\",\"MemUsage\":\"72.8MiB / 256MiB\",\"Name\":\"mtusi-lab6-otel-collector-1\",\"NetIO\":\"7.32MB / 2.49MB\",\"PIDs\":\"15\"}", + "{\"BlockIO\":\"0B / 0B\",\"CPUPerc\":\"0.43%\",\"Container\":\"6e21c63012d160a406e28a342d668de0be55107aa16228ed9d5dc15c34b447be\",\"ID\":\"6e21c63012d1\",\"MemPerc\":\"27.52%\",\"MemUsage\":\"70.46MiB / 256MiB\",\"Name\":\"mtusi-lab6-otel-collector-1\",\"NetIO\":\"7.48MB / 2.53MB\",\"PIDs\":\"15\"}", + "{\"BlockIO\":\"0B / 0B\",\"CPUPerc\":\"0.42%\",\"Container\":\"6e21c63012d160a406e28a342d668de0be55107aa16228ed9d5dc15c34b447be\",\"ID\":\"6e21c63012d1\",\"MemPerc\":\"27.95%\",\"MemUsage\":\"71.56MiB / 256MiB\",\"Name\":\"mtusi-lab6-otel-collector-1\",\"NetIO\":\"7.64MB / 2.58MB\",\"PIDs\":\"15\"}", + "{\"BlockIO\":\"0B / 0B\",\"CPUPerc\":\"0.20%\",\"Container\":\"6e21c63012d160a406e28a342d668de0be55107aa16228ed9d5dc15c34b447be\",\"ID\":\"6e21c63012d1\",\"MemPerc\":\"27.96%\",\"MemUsage\":\"71.57MiB / 256MiB\",\"Name\":\"mtusi-lab6-otel-collector-1\",\"NetIO\":\"7.81MB / 2.62MB\",\"PIDs\":\"15\"}" + ], + "collector_limit_bytes": 268435456, + "oom_killed": false, + "failed_exports_before": [ + { + "metric": { + "__name__": "otelcol_exporter_send_failed_log_records_total", + "exporter": "otlphttp/loki", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1795100986.923, + "6" + ] + } + ], + "failed_exports_after": [ + { + "metric": { + "__name__": "otelcol_exporter_send_failed_log_records_total", + "exporter": "otlphttp/loki", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1795101059.133, + "12" + ] + } + ] + }, + "provisioned_dashboard_queries": { + "passed": true, + "panels": 19 + } + }, + "dashboard": "messenger-lab6", + "passed": true, + "duration_seconds": 85.23 +} diff --git a/evidence/lab6-checks.txt b/evidence/lab6-checks.txt new file mode 100644 index 0000000..3c9baa7 --- /dev/null +++ b/evidence/lab6-checks.txt @@ -0,0 +1,102 @@ +{ + "utc": "2026-11-19T15:09:33.967862+00:00", + "checks": { + "retry_trace_and_logs": { + "passed": true, + "trace_id": "93e1177f9568f5bbfc0b6967a1324a84", + "request_id": "422b23e9-c8f9-4810-ad09-1cb99b6c9576", + "job_id": "fd4d7cce-b0dc-4edd-9938-61230afa7370", + "span_count": 59 + }, + "metrics_and_cardinality": { + "passed": true, + "series": 1192 + }, + "backend_outage": { + "passed": true, + "backend": "loki", + "successful_writes": 24, + "outage_seconds": 51.39, + "missing_request_logs": 6, + "tested_request_ids": [ + "a5835a4b-40b6-4383-90a3-82d07b5bdcfb", + "ee8b61c0-9acc-4ff4-a0fd-f4a7d84e29a2", + "bb492157-8ffd-49a6-84ef-c2b17379bc8e", + "6a076038-13e0-415d-b961-254cb7d48194", + "59ab7cdd-f319-4828-a0b2-4bb9ecef9464", + "0b472396-a1f4-453c-aff2-0a8e2650f3e3", + "b0916ed0-b218-423a-ac39-1c6e904300e6", + "bcd32c58-953b-4759-ac8b-e87fc66c4fd4", + "0a065b55-3fac-46fc-9d92-c67f8ba6e409", + "11d46ead-2941-4279-aa86-0abf111bcaba", + "f6807acd-4e5f-4bef-9f35-93b2ac581390", + "fec97823-5700-4fab-a18a-8a98f11f5c4e", + "a03b0d52-34ea-4b0a-86f2-ac408a0a107c", + "6d44ee0d-f039-406b-b0f7-4ceb7743d020", + "efc0b1d3-e889-4ab4-a169-0369c4cb260b", + "0ae460c2-6b26-4f5c-8959-ecb48a61ca92", + "0f59b3eb-2187-412b-b569-c2373748020e", + "086464d0-5b79-4c9d-aa95-e08af7858c43", + "4c42e246-b973-4e79-8a37-be114b255b80", + "2065d6ba-c0f8-43a5-999f-55ec5299d855", + "8ccb2503-c32f-4b3d-9763-2d9455f13a99", + "979ef391-bc49-42af-95b8-194e028dd796", + "fe3a4f81-114e-4055-b160-7f47a15acfff", + "e6019f30-5b26-4bc1-a630-bcfbbeb6ba96" + ], + "missing_request_ids": [ + "a5835a4b-40b6-4383-90a3-82d07b5bdcfb", + "ee8b61c0-9acc-4ff4-a0fd-f4a7d84e29a2", + "bb492157-8ffd-49a6-84ef-c2b17379bc8e", + "6a076038-13e0-415d-b961-254cb7d48194", + "59ab7cdd-f319-4828-a0b2-4bb9ecef9464", + "0b472396-a1f4-453c-aff2-0a8e2650f3e3" + ], + "collector_memory_samples": [ + "{\"BlockIO\":\"0B / 0B\",\"CPUPerc\":\"0.12%\",\"Container\":\"6e21c63012d160a406e28a342d668de0be55107aa16228ed9d5dc15c34b447be\",\"ID\":\"6e21c63012d1\",\"MemPerc\":\"28.44%\",\"MemUsage\":\"72.8MiB / 256MiB\",\"Name\":\"mtusi-lab6-otel-collector-1\",\"NetIO\":\"7.32MB / 2.49MB\",\"PIDs\":\"15\"}", + "{\"BlockIO\":\"0B / 0B\",\"CPUPerc\":\"0.43%\",\"Container\":\"6e21c63012d160a406e28a342d668de0be55107aa16228ed9d5dc15c34b447be\",\"ID\":\"6e21c63012d1\",\"MemPerc\":\"27.52%\",\"MemUsage\":\"70.46MiB / 256MiB\",\"Name\":\"mtusi-lab6-otel-collector-1\",\"NetIO\":\"7.48MB / 2.53MB\",\"PIDs\":\"15\"}", + "{\"BlockIO\":\"0B / 0B\",\"CPUPerc\":\"0.42%\",\"Container\":\"6e21c63012d160a406e28a342d668de0be55107aa16228ed9d5dc15c34b447be\",\"ID\":\"6e21c63012d1\",\"MemPerc\":\"27.95%\",\"MemUsage\":\"71.56MiB / 256MiB\",\"Name\":\"mtusi-lab6-otel-collector-1\",\"NetIO\":\"7.64MB / 2.58MB\",\"PIDs\":\"15\"}", + "{\"BlockIO\":\"0B / 0B\",\"CPUPerc\":\"0.20%\",\"Container\":\"6e21c63012d160a406e28a342d668de0be55107aa16228ed9d5dc15c34b447be\",\"ID\":\"6e21c63012d1\",\"MemPerc\":\"27.96%\",\"MemUsage\":\"71.57MiB / 256MiB\",\"Name\":\"mtusi-lab6-otel-collector-1\",\"NetIO\":\"7.81MB / 2.62MB\",\"PIDs\":\"15\"}" + ], + "collector_limit_bytes": 268435456, + "oom_killed": false, + "failed_exports_before": [ + { + "metric": { + "__name__": "otelcol_exporter_send_failed_log_records_total", + "exporter": "otlphttp/loki", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1795100986.923, + "6" + ] + } + ], + "failed_exports_after": [ + { + "metric": { + "__name__": "otelcol_exporter_send_failed_log_records_total", + "exporter": "otlphttp/loki", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1795101059.133, + "12" + ] + } + ] + }, + "provisioned_dashboard_queries": { + "passed": true, + "panels": 19 + } + }, + "dashboard": "messenger-lab6", + "passed": true, + "duration_seconds": 85.23 +} diff --git a/evidence/lab6-correlated-logs.json b/evidence/lab6-correlated-logs.json new file mode 100644 index 0000000..85172c7 --- /dev/null +++ b/evidence/lab6-correlated-logs.json @@ -0,0 +1,306 @@ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "attachment_id": "0eb04e8c-8e7b-42a9-ad63-8fca3390d11c", + "deployment_environment_name": "lab", + "detected_level": "INFO", + "event": "attachment.accepted", + "flags": "3", + "job_id": "fd4d7cce-b0dc-4edd-9938-61230afa7370", + "scope_name": "messenger", + "scope_version": "6.0.0", + "service_instance_id": "166e3f521d4f:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "e04c2c74d09fa066", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1", + "trace_id": "93e1177f9568f5bbfc0b6967a1324a84" + }, + "values": [ + [ + "1795100974767540043", + "{\"timestamp\": \"2026-11-19T15:09:34.767497+00:00\", \"severity\": \"INFO\", \"service.name\": \"messenger-api\", \"event\": \"attachment.accepted\", \"trace_id\": \"93e1177f9568f5bbfc0b6967a1324a84\", \"span_id\": \"e04c2c74d09fa066\", \"job_id\": \"fd4d7cce-b0dc-4edd-9938-61230afa7370\", \"attachment_id\": \"0eb04e8c-8e7b-42a9-ad63-8fca3390d11c\"}" + ] + ] + }, + { + "stream": { + "attempt": "1", + "deployment_environment_name": "lab", + "detected_level": "ERROR", + "error_code": "storage_unavailable", + "event": "job.retry", + "flags": "3", + "job_id": "fd4d7cce-b0dc-4edd-9938-61230afa7370", + "scope_name": "messenger", + "scope_version": "6.0.0", + "service_instance_id": "4a4dd93b6332:8", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "f9ff281d3dbfaedd", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1", + "trace_id": "93e1177f9568f5bbfc0b6967a1324a84" + }, + "values": [ + [ + "1795100977543203752", + "{\"timestamp\": \"2026-11-19T15:09:37.543167+00:00\", \"severity\": \"ERROR\", \"service.name\": \"messenger-worker\", \"event\": \"job.retry\", \"trace_id\": \"93e1177f9568f5bbfc0b6967a1324a84\", \"span_id\": \"f9ff281d3dbfaedd\", \"job_id\": \"fd4d7cce-b0dc-4edd-9938-61230afa7370\", \"attempt\": 1, \"error_code\": \"storage_unavailable\"}" + ] + ] + }, + { + "stream": { + "attempt": "1", + "deployment_environment_name": "lab", + "detected_level": "ERROR", + "event": "job.finished", + "flags": "3", + "job_id": "fd4d7cce-b0dc-4edd-9938-61230afa7370", + "outcome": "retry", + "scope_name": "messenger", + "scope_version": "6.0.0", + "service_instance_id": "4a4dd93b6332:8", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "2bb32cd3759889ae", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1", + "trace_id": "93e1177f9568f5bbfc0b6967a1324a84" + }, + "values": [ + [ + "1795100977544205419", + "{\"timestamp\": \"2026-11-19T15:09:37.544179+00:00\", \"severity\": \"ERROR\", \"service.name\": \"messenger-worker\", \"event\": \"job.finished\", \"trace_id\": \"93e1177f9568f5bbfc0b6967a1324a84\", \"span_id\": \"2bb32cd3759889ae\", \"job_id\": \"fd4d7cce-b0dc-4edd-9938-61230afa7370\", \"attempt\": 1, \"outcome\": \"retry\"}" + ] + ] + }, + { + "stream": { + "attempt": "2", + "deployment_environment_name": "lab", + "detected_level": "INFO", + "event": "job.finished", + "flags": "3", + "job_id": "fd4d7cce-b0dc-4edd-9938-61230afa7370", + "outcome": "ready", + "scope_name": "messenger", + "scope_version": "6.0.0", + "service_instance_id": "4a4dd93b6332:9", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "26b0193c185f8761", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1", + "trace_id": "93e1177f9568f5bbfc0b6967a1324a84" + }, + "values": [ + [ + "1795100979222424711", + "{\"timestamp\": \"2026-11-19T15:09:39.222385+00:00\", \"severity\": \"INFO\", \"service.name\": \"messenger-worker\", \"event\": \"job.finished\", \"trace_id\": \"93e1177f9568f5bbfc0b6967a1324a84\", \"span_id\": \"26b0193c185f8761\", \"job_id\": \"fd4d7cce-b0dc-4edd-9938-61230afa7370\", \"attempt\": 2, \"outcome\": \"ready\"}" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "INFO", + "duration_seconds": "0.009846", + "event": "http.request", + "flags": "3", + "method": "POST", + "request_id": "422b23e9-c8f9-4810-ad09-1cb99b6c9576", + "route": "/api/v1/chats/{chat_id}/attachments", + "scope_name": "messenger", + "scope_version": "6.0.0", + "service_instance_id": "166e3f521d4f:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "e04c2c74d09fa066", + "status": "202", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1", + "trace_id": "93e1177f9568f5bbfc0b6967a1324a84" + }, + "values": [ + [ + "1795100974767795501", + "{\"timestamp\": \"2026-11-19T15:09:34.767775+00:00\", \"severity\": \"INFO\", \"service.name\": \"messenger-api\", \"event\": \"http.request\", \"trace_id\": \"93e1177f9568f5bbfc0b6967a1324a84\", \"span_id\": \"e04c2c74d09fa066\", \"method\": \"POST\", \"route\": \"/api/v1/chats/{chat_id}/attachments\", \"status\": 202, \"duration_seconds\": 0.009846, \"request_id\": \"422b23e9-c8f9-4810-ad09-1cb99b6c9576\"}" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 80485548, + "linesProcessedPerSecond": 110946, + "totalBytesProcessed": 1706250, + "totalLinesProcessed": 2352, + "execTime": 0.021199, + "queueTime": 0.000413, + "subqueries": 0, + "totalEntriesReturned": 5, + "splits": 2, + "shards": 2, + "totalPostFilterLines": 5, + "totalStructuredMetadataBytesProcessed": 853184 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 2, + "totalChunksMatched": 2, + "totalBatches": 3, + "totalLinesSent": 5, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 114091, + "headChunkLines": 228, + "decompressedBytes": 1592159, + "decompressedLines": 2124, + "compressedBytes": 190062, + "totalDuplicates": 0, + "postFilterLines": 5, + "headChunkStructuredMetadataBytes": 34104, + "decompressedStructuredMetadataBytes": 819080 + }, + "chunkRefsFetchTime": 180667, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false + } + } + } +} diff --git a/evidence/lab6-dashboard-checks.json b/evidence/lab6-dashboard-checks.json new file mode 100644 index 0000000..4bf952a --- /dev/null +++ b/evidence/lab6-dashboard-checks.json @@ -0,0 +1,100 @@ +{ + "passed": true, + "panels": [ + { + "panel": "Business API RPS", + "source": "prometheus", + "passed": true + }, + { + "panel": "API 5xx / all business requests", + "source": "prometheus", + "passed": true + }, + { + "panel": "API latency: aggregated histogram", + "source": "prometheus", + "passed": true + }, + { + "panel": "Time from acceptance to ready: completed jobs", + "source": "prometheus", + "passed": true + }, + { + "panel": "Job attempts by outcome", + "source": "prometheus", + "passed": true + }, + { + "panel": "Worker processing p95", + "source": "prometheus", + "passed": true + }, + { + "panel": "Outstanding jobs (DB source of truth)", + "source": "prometheus", + "passed": true + }, + { + "panel": "Oldest unfinished job age", + "source": "prometheus", + "passed": true + }, + { + "panel": "DB pool used / waiting", + "source": "prometheus", + "passed": true + }, + { + "panel": "Worker busy / capacity", + "source": "prometheus", + "passed": true + }, + { + "panel": "Process CPU: cores consumed", + "source": "prometheus", + "passed": true + }, + { + "panel": "Process RSS", + "source": "prometheus", + "passed": true + }, + { + "panel": "API availability SLO: >= 99% / 1h", + "source": "prometheus", + "passed": true + }, + { + "panel": "Ready SLO: >= 95% within 30s / 1h", + "source": "prometheus", + "passed": true + }, + { + "panel": "API burn rate (budget / hour)", + "source": "prometheus", + "passed": true + }, + { + "panel": "Ready burn rate (budget / hour)", + "source": "prometheus", + "passed": true + }, + { + "panel": "Local alert states", + "source": "prometheus", + "passed": true + }, + { + "panel": "API / worker structured logs (click TraceID)", + "source": "loki", + "passed": true + }, + { + "panel": "Recent real worker traces", + "source": "tempo", + "passed": true + } + ] +} diff --git a/evidence/lab6-db-incident-logs.json b/evidence/lab6-db-incident-logs.json new file mode 100644 index 0000000..8288f59 --- /dev/null +++ b/evidence/lab6-db-incident-logs.json @@ -0,0 +1,191 @@ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "ERROR", + "duration_seconds": "2.003678", + "event": "http.request", + "flags": "3", + "method": "POST", + "request_id": "3c5e5b14-5825-41b8-a227-93cba420a532", + "route": "/api/v1/chats/{chat_id}/messages", + "scope_name": "messenger", + "scope_version": "6.0.0", + "service_instance_id": "2a50b728ce10:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "severity_number": "17", + "severity_text": "ERROR", + "span_id": "3cec0bb07faa77ff", + "status": "503", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1", + "trace_id": "f124d93f84dd25c5af978e78e2028646" + }, + "values": [ + [ + "1795102249376144341", + "{\"timestamp\": \"2026-11-19T15:30:49.376082+00:00\", \"severity\": \"ERROR\", \"service.name\": \"messenger-api\", \"event\": \"http.request\", \"trace_id\": \"f124d93f84dd25c5af978e78e2028646\", \"span_id\": \"3cec0bb07faa77ff\", \"method\": \"POST\", \"route\": \"/api/v1/chats/{chat_id}/messages\", \"status\": 503, \"duration_seconds\": 2.003678, \"request_id\": \"3c5e5b14-5825-41b8-a227-93cba420a532\"}" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 61790648, + "linesProcessedPerSecond": 82173, + "totalBytesProcessed": 809854, + "totalLinesProcessed": 1077, + "execTime": 0.013106, + "queueTime": 0.000136, + "subqueries": 0, + "totalEntriesReturned": 1, + "splits": 0, + "shards": 1, + "totalPostFilterLines": 1, + "totalStructuredMetadataBytesProcessed": 400756 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 1, + "totalChunksMatched": 1, + "totalBatches": 2, + "totalLinesSent": 1, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 23978, + "headChunkLines": 47, + "decompressedBytes": 785876, + "decompressedLines": 1030, + "compressedBytes": 95894, + "totalDuplicates": 0, + "postFilterLines": 1, + "headChunkStructuredMetadataBytes": 6976, + "decompressedStructuredMetadataBytes": 393780 + }, + "chunkRefsFetchTime": 72000, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false + } + } + } +} diff --git a/evidence/lab6-db-incident-trace.json b/evidence/lab6-db-incident-trace.json new file mode 100644 index 0000000..3226eb3 --- /dev/null +++ b/evidence/lab6-db-incident-trace.json @@ -0,0 +1,275 @@ +{ + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "2a50b728ce10:1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "6.0.0" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "6.0.0" + }, + "spans": [ + { + "traceId": "8STZP4TdJcWvl4544gKGRg==", + "spanId": "6YhkxV7CY4k=", + "parentSpanId": "zAWgCBMyg3I=", + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102247372852382", + "endTimeUnixNano": "1795102247372984882", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "8STZP4TdJcWvl4544gKGRg==", + "spanId": "9dARsClEhgI=", + "parentSpanId": "zAWgCBMyg3I=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102247373056215", + "endTimeUnixNano": "1795102247373325590", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "8STZP4TdJcWvl4544gKGRg==", + "spanId": "zAWgCBMyg3I=", + "parentSpanId": "POwLsH+qd/8=", + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795102247372781257", + "endTimeUnixNano": "1795102247373442673", + "status": {} + }, + { + "traceId": "8STZP4TdJcWvl4544gKGRg==", + "spanId": "HOilfsT0R4s=", + "parentSpanId": "ZMWryXhMXeM=", + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102247374040798", + "endTimeUnixNano": "1795102247374149048", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "8STZP4TdJcWvl4544gKGRg==", + "spanId": "tNH67JGBzzQ=", + "parentSpanId": "ZMWryXhMXeM=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102247374219298", + "endTimeUnixNano": "1795102247374661423", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "8STZP4TdJcWvl4544gKGRg==", + "spanId": "pO76DIPXzps=", + "parentSpanId": "ZMWryXhMXeM=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102247374726007", + "endTimeUnixNano": "1795102247374869840", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "8STZP4TdJcWvl4544gKGRg==", + "spanId": "bxfNsCS1EJs=", + "parentSpanId": "ZMWryXhMXeM=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102247374927173", + "endTimeUnixNano": "1795102249375042508", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "_WaitTimeout" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "8STZP4TdJcWvl4544gKGRg==", + "spanId": "ZMWryXhMXeM=", + "parentSpanId": "POwLsH+qd/8=", + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795102247373987923", + "endTimeUnixNano": "1795102249375157091", + "attributes": [ + { + "key": "error.type", + "value": { + "stringValue": "_WaitTimeout" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "8STZP4TdJcWvl4544gKGRg==", + "spanId": "POwLsH+qd/8=", + "name": "POST /api/v1/chats/{chat_id}/messages", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795102247372424840", + "endTimeUnixNano": "1795102249376209799", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/messages" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + } + ] + } + ] + } + ] +} diff --git a/evidence/lab6-lab5-regression.json b/evidence/lab6-lab5-regression.json new file mode 100644 index 0000000..47e57cb --- /dev/null +++ b/evidence/lab6-lab5-regression.json @@ -0,0 +1,795 @@ +{ + "utc": "2026-11-19T15:09:44.028971+00:00", + "command": "./scripts/check-docker.sh", + "project": "lab5-check-c81d43a829", + "host": "Darwin arm64", + "source_commit": "470d9d6f1e4fe4f594e37c2ba59550ee5ca54fc1", + "working_tree": true, + "configuration": { + "concurrency": 2, + "prefetch": 1, + "attempts": 5, + "timeout_seconds": 5, + "lease_seconds": 15, + "backoff_base_seconds": 0.15 + }, + "checks": { + "topology_and_limits": { + "passed": true, + "api_replicas": 2, + "broker": "Valkey Streams with AOF fsync always", + "worker_processes": 2 + }, + "public_smoke_and_schema": { + "passed": true + }, + "student_api_regressions": { + "passed": true + }, + "worker_offline_api_broker_restart": { + "passed": true, + "attachment_id": "ef3471d3-538e-4d3f-8406-edfb2e78d803", + "job_id": "29387d95-7e9f-4d86-b645-024cb36049cf", + "message_id": "4ae76a83-fddf-4d6a-aa4c-1dffe8910540", + "variants": { + "thumbnail": { + "size": [ + 256, + 160 + ], + "bytes": 5813 + }, + "crop": { + "size": [ + 128, + 128 + ], + "bytes": 2462 + }, + "optimized": { + "size": [ + 320, + 200 + ], + "bytes": 2462 + } + } + }, + "outbox_crash_before_publish": { + "passed": true, + "job_id": "630dde85-21c6-4a82-bcee-5ea7aafa21b8", + "automatic_recovery": true + }, + "outbox_crash_after_publish": { + "passed": true, + "job_id": "5c9d0c57-07b5-41d9-951e-d07ed3ce2126", + "automatic_recovery": true + }, + "worker_crash_after_objects": { + "passed": true, + "job_id": "e7e67187-45e2-49ad-b845-7c80952abaed", + "attempt": 2, + "canonical_objects": 4, + "duplicate_noop": true + }, + "worker_crash_before_ack": { + "passed": true, + "job_id": "646cabc2-c06f-43a9-98ed-97a0ed430eef", + "attempt": 1, + "canonical_objects": 4, + "duplicate_noop": true + }, + "bounded_retry_failed_registry_replay": { + "passed": true, + "job_id": "9f0223e7-c7f7-41ce-a6b6-a6b2a6827db7", + "max_automatic_attempts": 5, + "audit": [ + { + "event": "processing", + "attempt": 1, + "error_code": null, + "at": "2026-11-19T15:11:43.910272+00:00" + }, + { + "event": "retry", + "attempt": 1, + "error_code": "storage_unavailable", + "at": "2026-11-19T15:11:43.913931+00:00" + }, + { + "event": "processing", + "attempt": 2, + "error_code": null, + "at": "2026-11-19T15:11:44.162093+00:00" + }, + { + "event": "retry", + "attempt": 2, + "error_code": "storage_unavailable", + "at": "2026-11-19T15:11:44.163784+00:00" + }, + { + "event": "processing", + "attempt": 3, + "error_code": null, + "at": "2026-11-19T15:11:44.672423+00:00" + }, + { + "event": "retry", + "attempt": 3, + "error_code": "storage_unavailable", + "at": "2026-11-19T15:11:44.674937+00:00" + }, + { + "event": "processing", + "attempt": 4, + "error_code": null, + "at": "2026-11-19T15:11:45.176057+00:00" + }, + { + "event": "retry", + "attempt": 4, + "error_code": "storage_unavailable", + "at": "2026-11-19T15:11:45.17712+00:00" + }, + { + "event": "processing", + "attempt": 5, + "error_code": null, + "at": "2026-11-19T15:11:45.941529+00:00" + }, + { + "event": "failed", + "attempt": 5, + "error_code": "retry_exhausted", + "at": "2026-11-19T15:11:45.944804+00:00" + }, + { + "event": "operator_replay", + "attempt": 5, + "error_code": null, + "at": "2026-11-19T15:11:47.029399+00:00" + }, + { + "event": "processing", + "attempt": 6, + "error_code": null, + "at": "2026-11-19T15:11:47.203814+00:00" + }, + { + "event": "ready", + "attempt": 6, + "error_code": null, + "at": "2026-11-19T15:11:47.601272+00:00" + } + ], + "ready_replay_rejected": true + }, + "permanent_error": { + "passed": true, + "job_id": "28022e30-0cd6-4df1-a41b-56905cf4feda", + "attempt": 1, + "error_code": "invalid_image" + }, + "image_subprocess_timeout": { + "passed": true, + "job_id": "c0f7e1b2-59a1-496e-a60a-8ed3fdb19f59", + "timeout_seconds": 5, + "task_memory_mb": 640, + "recovered": true + }, + "image_subprocess_memory": { + "passed": true, + "job_id": "f1e4caee-86d7-4185-9787-92687c0650a3", + "timeout_seconds": 5, + "task_memory_mb": 640, + "recovered": true + }, + "image_semantics": { + "passed": true, + "cases": [ + { + "case": "small_alpha_on_white", + "variants": { + "thumbnail": { + "size": [ + 17, + 9 + ], + "bytes": 635 + }, + "crop": { + "size": [ + 128, + 128 + ], + "bytes": 883 + }, + "optimized": { + "size": [ + 17, + 9 + ], + "bytes": 56 + } + } + }, + { + "case": "exif_orientation_6_and_floor", + "variants": { + "thumbnail": { + "size": [ + 160, + 256 + ], + "bytes": 1270 + }, + "crop": { + "size": [ + 128, + 128 + ], + "bytes": 886 + }, + "optimized": { + "size": [ + 201, + 321 + ], + "bytes": 210 + } + } + } + ] + }, + "staging_cleanup": { + "passed": true, + "active_preserved": true, + "orphan_removed": true, + "ready_removed": true, + "expired_failed_removed": true, + "expired_replay_rejected": true + }, + "broker_offline_outbox": { + "passed": true, + "attachment_id": "28321e5b-8b43-471a-973d-9b55cbc07b39", + "recovered": true + }, + "public_persistence": { + "passed": true, + "restart_proof": { + "passed": true, + "before": { + "ea2b13aebb92a1ea2741de72078489e6b05ebcc41f1ca74a2604c0ad7d355983": { + "service": "api", + "started_at": "2026-11-19T15:10:30.841890222Z", + "volumes": [] + }, + "67eade4e281a4cfed3f55d87ab398097e12ee12155f82cbe19eb240a4e92b41a": { + "service": "api", + "started_at": "2026-11-19T15:10:30.809775805Z", + "volumes": [] + }, + "d675c92d195a1bb37db5860c3cf49a2a918eb3e98cfd9c855f8f89931138724b": { + "service": "broker", + "started_at": "2026-11-19T15:12:24.444290385Z", + "volumes": [ + "lab5-check-c81d43a829_broker_data" + ] + }, + "f9b2e0b6e448dda6d845d9715d64e1216fc0f3f58c3e05ec22de30d305c2d3e6": { + "service": "cleanup", + "started_at": "2026-11-19T15:10:11.903050713Z", + "volumes": [] + }, + "e4abde6d7d326be2a1425d81667ab967dd655ce1eb39c1b0c1bd2f9956f3e91f": { + "service": "grafana", + "started_at": "2026-11-19T15:09:46.76721934Z", + "volumes": [ + "lab5-check-c81d43a829_grafana_data" + ] + }, + "1e9b604ee9fe91572a4efd3b629a00819d86ab2b45ed1f04a1df66428cfe1920": { + "service": "lb", + "started_at": "2026-11-19T15:10:17.66400909Z", + "volumes": [] + }, + "487c1937cced6d3b86d6a67cae2b76fdfc2b534e6cf15b6cf2242b96b19c8af4": { + "service": "loki", + "started_at": "2026-11-19T15:09:46.088598506Z", + "volumes": [ + "lab5-check-c81d43a829_loki_data" + ] + }, + "f31970f75546cfcf6c390fad538eb9a7ebc25d26c481caef09bf679a5af8e8d9": { + "service": "otel-collector", + "started_at": "2026-11-19T15:09:46.409972798Z", + "volumes": [] + }, + "82576012c7b7b5a5a352207679f4c327fe50cbd5c56337de33c16ec15b2b0501": { + "service": "postgres", + "started_at": "2026-11-19T15:09:46.08945334Z", + "volumes": [ + "lab5-check-c81d43a829_postgres_data" + ] + }, + "6e305e5121dc3113f906c5689d1006091714d72b452acaabd07e3d366f1d13f6": { + "service": "prometheus", + "started_at": "2026-11-19T15:09:46.555234256Z", + "volumes": [ + "lab5-check-c81d43a829_prometheus_data" + ] + }, + "fb35c350ab7fcff516b44e4f6933b1e293c896e6cee9bf29d06e71ba99a0f931": { + "service": "publisher", + "started_at": "2026-11-19T15:12:10.913909129Z", + "volumes": [] + }, + "8cdcd8bdac73e3bc827ef7bc5a77bad0696c1023cd330eb10206dcc6b29529cf": { + "service": "s3", + "started_at": "2026-11-19T15:10:08.720162878Z", + "volumes": [ + "d6e81fdf6ec0a4bd30cb0461085c08461b951f8ae9c696222d04b21152ff6560" + ] + }, + "053f3a4b4e5cd9b28510f08e9bd8d07374c586af075cb62d5c5b49f62af5a6ac": { + "service": "s3-storage", + "started_at": "2026-11-19T15:09:46.086474548Z", + "volumes": [ + "lab5-check-c81d43a829_s3_data" + ] + }, + "84473a25dbb67b410a0db8d8b0e029b84b264b3238da1961c7693f13d9e99634": { + "service": "tempo", + "started_at": "2026-11-19T15:09:46.086451215Z", + "volumes": [ + "lab5-check-c81d43a829_tempo_data" + ] + }, + "7f3ff970cf57bedf3a8800a6bed5cc3a83f749dab07d9ac42425d14c7eafee80": { + "service": "valkey", + "started_at": "2026-11-19T15:09:46.08714634Z", + "volumes": [ + "lab5-check-c81d43a829_valkey_data" + ] + }, + "521131dedb68623718c9d5d5c8dc8b0924dbfe6ec90acd3b21e08189a9a1a1e5": { + "service": "worker", + "started_at": "2026-11-19T15:12:21.892301301Z", + "volumes": [] + } + }, + "after": { + "ea2b13aebb92a1ea2741de72078489e6b05ebcc41f1ca74a2604c0ad7d355983": { + "service": "api", + "started_at": "2026-11-19T15:12:30.097473346Z", + "volumes": [] + }, + "67eade4e281a4cfed3f55d87ab398097e12ee12155f82cbe19eb240a4e92b41a": { + "service": "api", + "started_at": "2026-11-19T15:12:30.048159138Z", + "volumes": [] + }, + "d675c92d195a1bb37db5860c3cf49a2a918eb3e98cfd9c855f8f89931138724b": { + "service": "broker", + "started_at": "2026-11-19T15:12:28.168864679Z", + "volumes": [ + "lab5-check-c81d43a829_broker_data" + ] + }, + "f9b2e0b6e448dda6d845d9715d64e1216fc0f3f58c3e05ec22de30d305c2d3e6": { + "service": "cleanup", + "started_at": "2026-11-19T15:12:35.75098096Z", + "volumes": [] + }, + "e4abde6d7d326be2a1425d81667ab967dd655ce1eb39c1b0c1bd2f9956f3e91f": { + "service": "grafana", + "started_at": "2026-11-19T15:12:28.016722095Z", + "volumes": [ + "lab5-check-c81d43a829_grafana_data" + ] + }, + "1e9b604ee9fe91572a4efd3b629a00819d86ab2b45ed1f04a1df66428cfe1920": { + "service": "lb", + "started_at": "2026-11-19T15:12:27.799505637Z", + "volumes": [] + }, + "487c1937cced6d3b86d6a67cae2b76fdfc2b534e6cf15b6cf2242b96b19c8af4": { + "service": "loki", + "started_at": "2026-11-19T15:12:29.867573388Z", + "volumes": [ + "lab5-check-c81d43a829_loki_data" + ] + }, + "f31970f75546cfcf6c390fad538eb9a7ebc25d26c481caef09bf679a5af8e8d9": { + "service": "otel-collector", + "started_at": "2026-11-19T15:12:29.704735929Z", + "volumes": [] + }, + "82576012c7b7b5a5a352207679f4c327fe50cbd5c56337de33c16ec15b2b0501": { + "service": "postgres", + "started_at": "2026-11-19T15:12:28.084774804Z", + "volumes": [ + "lab5-check-c81d43a829_postgres_data" + ] + }, + "6e305e5121dc3113f906c5689d1006091714d72b452acaabd07e3d366f1d13f6": { + "service": "prometheus", + "started_at": "2026-11-19T15:12:28.052574179Z", + "volumes": [ + "lab5-check-c81d43a829_prometheus_data" + ] + }, + "fb35c350ab7fcff516b44e4f6933b1e293c896e6cee9bf29d06e71ba99a0f931": { + "service": "publisher", + "started_at": "2026-11-19T15:12:35.770825668Z", + "volumes": [] + }, + "8cdcd8bdac73e3bc827ef7bc5a77bad0696c1023cd330eb10206dcc6b29529cf": { + "service": "s3", + "started_at": "2026-11-19T15:12:28.115603887Z", + "volumes": [ + "d6e81fdf6ec0a4bd30cb0461085c08461b951f8ae9c696222d04b21152ff6560" + ] + }, + "053f3a4b4e5cd9b28510f08e9bd8d07374c586af075cb62d5c5b49f62af5a6ac": { + "service": "s3-storage", + "started_at": "2026-11-19T15:12:35.714312418Z", + "volumes": [ + "lab5-check-c81d43a829_s3_data" + ] + }, + "84473a25dbb67b410a0db8d8b0e029b84b264b3238da1961c7693f13d9e99634": { + "service": "tempo", + "started_at": "2026-11-19T15:12:28.142091095Z", + "volumes": [ + "lab5-check-c81d43a829_tempo_data" + ] + }, + "7f3ff970cf57bedf3a8800a6bed5cc3a83f749dab07d9ac42425d14c7eafee80": { + "service": "valkey", + "started_at": "2026-11-19T15:12:28.230646637Z", + "volumes": [ + "lab5-check-c81d43a829_valkey_data" + ] + }, + "521131dedb68623718c9d5d5c8dc8b0924dbfe6ec90acd3b21e08189a9a1a1e5": { + "service": "worker", + "started_at": "2026-11-19T15:12:27.946355887Z", + "volumes": [] + } + }, + "same_containers_and_volumes": true + } + }, + "public_api_failover": { + "passed": true + } + }, + "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", + "containers": [ + { + "id": "ea2b13aebb92a1ea2741de72078489e6b05ebcc41f1ca74a2604c0ad7d355983", + "service": "api", + "image": "mtusi-messenger:lab6", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/database_url", + "name": null + }, + { + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "destination": "/run/secrets/s3_secret_access_key", + "name": null + } + ] + }, + { + "id": "67eade4e281a4cfed3f55d87ab398097e12ee12155f82cbe19eb240a4e92b41a", + "service": "api", + "image": "mtusi-messenger:lab6", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/cache_url", + "name": null + }, + { + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "d675c92d195a1bb37db5860c3cf49a2a918eb3e98cfd9c855f8f89931138724b", + "service": "broker", + "image": "valkey/valkey:8.1.3", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/data", + "name": "lab5-check-c81d43a829_broker_data" + }, + { + "destination": "/run/secrets/broker_password", + "name": null + } + ] + }, + { + "id": "f9b2e0b6e448dda6d845d9715d64e1216fc0f3f58c3e05ec22de30d305c2d3e6", + "service": "cleanup", + "image": "mtusi-messenger:lab6", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/s3_access_key_id", + "name": null + }, + { + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "e4abde6d7d326be2a1425d81667ab967dd655ce1eb39c1b0c1bd2f9956f3e91f", + "service": "grafana", + "image": "grafana/grafana:12.0.2", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/etc/grafana/provisioning", + "name": null + }, + { + "destination": "/var/lib/grafana/dashboards", + "name": null + }, + { + "destination": "/var/lib/grafana", + "name": "lab5-check-c81d43a829_grafana_data" + } + ] + }, + { + "id": "1e9b604ee9fe91572a4efd3b629a00819d86ab2b45ed1f04a1df66428cfe1920", + "service": "lb", + "image": "haproxy:3.2.6-alpine", + "memory_bytes": 67108864, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/usr/local/etc/haproxy/haproxy.cfg", + "name": null + } + ] + }, + { + "id": "487c1937cced6d3b86d6a67cae2b76fdfc2b534e6cf15b6cf2242b96b19c8af4", + "service": "loki", + "image": "grafana/loki:3.5.1", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/etc/loki/config.yaml", + "name": null + }, + { + "destination": "/loki", + "name": "lab5-check-c81d43a829_loki_data" + } + ] + }, + { + "id": "02572452b7c401e702c6ad13daac234d9e3c069bb791e44770a7f7fc11dca885", + "service": "migrate", + "image": "mtusi-messenger:lab6", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/database_url", + "name": null + } + ] + }, + { + "id": "f31970f75546cfcf6c390fad538eb9a7ebc25d26c481caef09bf679a5af8e8d9", + "service": "otel-collector", + "image": "otel/opentelemetry-collector-contrib:0.123.0", + "memory_bytes": 268435456, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/etc/otelcol/config.yaml", + "name": null + } + ] + }, + { + "id": "82576012c7b7b5a5a352207679f4c327fe50cbd5c56337de33c16ec15b2b0501", + "service": "postgres", + "image": "postgres:17.6-bookworm", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/postgres_password", + "name": null + }, + { + "destination": "/var/lib/postgresql/data", + "name": "lab5-check-c81d43a829_postgres_data" + } + ] + }, + { + "id": "6e305e5121dc3113f906c5689d1006091714d72b452acaabd07e3d366f1d13f6", + "service": "prometheus", + "image": "prom/prometheus:v3.4.1", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/prometheus", + "name": "lab5-check-c81d43a829_prometheus_data" + }, + { + "destination": "/etc/prometheus/prometheus.yaml", + "name": null + }, + { + "destination": "/etc/prometheus/alerts.yaml", + "name": null + } + ] + }, + { + "id": "fb35c350ab7fcff516b44e4f6933b1e293c896e6cee9bf29d06e71ba99a0f931", + "service": "publisher", + "image": "mtusi-messenger:lab6", + "memory_bytes": 402653184, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/run/secrets/database_url", + "name": null + }, + { + "destination": "/run/secrets/broker_url", + "name": null + } + ] + }, + { + "id": "8cdcd8bdac73e3bc827ef7bc5a77bad0696c1023cd330eb10206dcc6b29529cf", + "service": "s3", + "image": "chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/run/secrets/s3_config", + "name": null + }, + { + "destination": "/data", + "name": "d6e81fdf6ec0a4bd30cb0461085c08461b951f8ae9c696222d04b21152ff6560" + } + ] + }, + { + "id": "e491085d55967c2cc9a4c907385fa87d8ee2e7d072ae77082a39ffde575cf0e8", + "service": "s3-bootstrap", + "image": "mtusi-messenger:lab6", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/run/secrets/s3_root_user", + "name": null + }, + { + "destination": "/run/secrets/s3_root_password", + "name": null + } + ] + }, + { + "id": "053f3a4b4e5cd9b28510f08e9bd8d07374c586af075cb62d5c5b49f62af5a6ac", + "service": "s3-storage", + "image": "chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/data", + "name": "lab5-check-c81d43a829_s3_data" + } + ] + }, + { + "id": "84473a25dbb67b410a0db8d8b0e029b84b264b3238da1961c7693f13d9e99634", + "service": "tempo", + "image": "grafana/tempo:2.8.1", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "mounts": [ + { + "destination": "/etc/tempo/config.yaml", + "name": null + }, + { + "destination": "/var/tempo", + "name": "lab5-check-c81d43a829_tempo_data" + } + ] + }, + { + "id": "7f3ff970cf57bedf3a8800a6bed5cc3a83f749dab07d9ac42425d14c7eafee80", + "service": "valkey", + "image": "valkey/valkey:8.1.3", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "mounts": [ + { + "destination": "/data", + "name": "lab5-check-c81d43a829_valkey_data" + } + ] + }, + { + "id": "521131dedb68623718c9d5d5c8dc8b0924dbfe6ec90acd3b21e08189a9a1a1e5", + "service": "worker", + "image": "mtusi-messenger:lab6", + "memory_bytes": 1610612736, + "nano_cpus": 2000000000, + "mounts": [ + { + "destination": "/run/secrets/s3_secret_access_key", + "name": null + }, + { + "destination": "/run/secrets/database_url", + "name": null + }, + { + "destination": "/run/secrets/broker_url", + "name": null + }, + { + "destination": "/run/secrets/s3_access_key_id", + "name": null + } + ] + } + ], + "passed": true +} diff --git a/evidence/lab6-lab5-regression.txt b/evidence/lab6-lab5-regression.txt new file mode 100644 index 0000000..d0cfe7b --- /dev/null +++ b/evidence/lab6-lab5-regression.txt @@ -0,0 +1,77 @@ +PASS: topology_and_limits {"api_replicas": 2, "broker": "Valkey Streams with AOF fsync always", "worker_processes": 2} +/Users/vanek/Desktop/VNPlabs/VNPlab6/mtusi2026lab6/.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +/Users/vanek/Desktop/VNPlabs/VNPlab6/mtusi2026lab6/.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +/Users/vanek/Desktop/VNPlabs/VNPlab6/mtusi2026lab6/.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:63012" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +PASS: public_smoke_and_schema {} +PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages. +test_chat_access (test_api.ApiTests.test_chat_access) ... ok +test_chat_pagination_and_cursor_owner (test_api.ApiTests.test_chat_pagination_and_cursor_owner) ... ok +test_cursor_cannot_be_used_for_another_chat_or_changed (test_api.ApiTests.test_cursor_cannot_be_used_for_another_chat_or_changed) ... ok +test_default_page_size (test_api.ApiTests.test_default_page_size) ... ok +test_duplicate_username (test_api.ApiTests.test_duplicate_username) ... ok +test_framework_errors_and_request_ids (test_api.ApiTests.test_framework_errors_and_request_ids) ... ok +test_identification_and_authentication_order (test_api.ApiTests.test_identification_and_authentication_order) ... ok +test_invalid_chat_members (test_api.ApiTests.test_invalid_chat_members) ... ok +test_invalid_chat_title (test_api.ApiTests.test_invalid_chat_title) ... ok +test_invalid_limits_and_cursors (test_api.ApiTests.test_invalid_limits_and_cursors) ... ok +test_invalid_message_fields (test_api.ApiTests.test_invalid_message_fields) ... ok +test_invalid_user_fields (test_api.ApiTests.test_invalid_user_fields) ... ok +test_malformed_json_and_encoding (test_api.ApiTests.test_malformed_json_and_encoding) ... ok +test_message_pagination_and_changing_limit (test_api.ApiTests.test_message_pagination_and_changing_limit) ... ok +test_unicode_and_text_preservation (test_api.ApiTests.test_unicode_and_text_preservation) ... ok +test_wrong_content_type (test_api.ApiTests.test_wrong_content_type) ... ok +test_access_and_authentication (test_attachments.AttachmentTests.test_access_and_authentication) ... ok +test_attachment_only_message_and_other_member (test_attachments.AttachmentTests.test_attachment_only_message_and_other_member) ... ok +test_chunked_limit_without_content_length (test_attachments.AttachmentTests.test_chunked_limit_without_content_length) ... ok +test_content_length_limit_before_body (test_attachments.AttachmentTests.test_content_length_limit_before_body) ... ok +test_corruption_and_mime_spoof (test_attachments.AttachmentTests.test_corruption_and_mime_spoof) ... ok +test_filename_rejected_before_storage (test_attachments.AttachmentTests.test_filename_rejected_before_storage) ... ok +test_foreign_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_foreign_and_unknown_attachment_ids) ... ok +test_idempotency_includes_ordered_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_ordered_attachment_ids) ... ok +test_invalid_attachment_lists (test_attachments.AttachmentTests.test_invalid_attachment_lists) ... ok +test_pixel_limit_before_decode (test_attachments.AttachmentTests.test_pixel_limit_before_decode) ... ok +test_png_and_jpeg_originals (test_attachments.AttachmentTests.test_png_and_jpeg_originals) ... ok +test_simultaneous_registration_of_one_username (test_concurrency.ConcurrencyTests.test_simultaneous_registration_of_one_username) ... ok +test_ten_clients_send_one_hundred_messages (test_concurrency.ConcurrencyTests.test_ten_clients_send_one_hundred_messages) ... ok +test_conflict_and_semantic_json_replay (test_idempotency.IdempotencyTests.test_conflict_and_semantic_json_replay) ... ok +test_key_format_and_missing_key (test_idempotency.IdempotencyTests.test_key_format_and_missing_key) ... ok +test_parallel_replay_across_replicas (test_idempotency.IdempotencyTests.test_parallel_replay_across_replicas) ... ok +test_replay_rechecks_session_and_access (test_idempotency.IdempotencyTests.test_replay_rechecks_session_and_access) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_logout_revokes_only_one_session (test_sessions.SessionTests.test_logout_revokes_only_one_session) ... ok +test_password_validation (test_sessions.SessionTests.test_password_validation) ... ok +test_sql_like_message_is_plain_text (test_sessions.SessionTests.test_sql_like_message_is_plain_text) ... ok +test_unicode_password_with_colon_and_private_response (test_sessions.SessionTests.test_unicode_password_with_colon_and_private_response) ... ok +test_wrong_password_and_basic_only_for_login (test_sessions.SessionTests.test_wrong_password_and_basic_only_for_login) ... ok + +---------------------------------------------------------------------- +Ran 39 tests in 7.761s + +OK +PASS: student_api_regressions {} +PASS: worker_offline_api_broker_restart {"attachment_id": "ef3471d3-538e-4d3f-8406-edfb2e78d803", "job_id": "29387d95-7e9f-4d86-b645-024cb36049cf", "message_id": "4ae76a83-fddf-4d6a-aa4c-1dffe8910540", "variants": {"thumbnail": {"size": [256, 160], "bytes": 5813}, "crop": {"size": [128, 128], "bytes": 2462}, "optimized": {"size": [320, 200], "bytes": 2462}}} +PASS: outbox_crash_before_publish {"job_id": "630dde85-21c6-4a82-bcee-5ea7aafa21b8", "automatic_recovery": true} +PASS: outbox_crash_after_publish {"job_id": "5c9d0c57-07b5-41d9-951e-d07ed3ce2126", "automatic_recovery": true} +PASS: worker_crash_after_objects {"job_id": "e7e67187-45e2-49ad-b845-7c80952abaed", "attempt": 2, "canonical_objects": 4, "duplicate_noop": true} +PASS: worker_crash_before_ack {"job_id": "646cabc2-c06f-43a9-98ed-97a0ed430eef", "attempt": 1, "canonical_objects": 4, "duplicate_noop": true} +PASS: bounded_retry_failed_registry_replay {"job_id": "9f0223e7-c7f7-41ce-a6b6-a6b2a6827db7", "max_automatic_attempts": 5, "audit": [{"event": "processing", "attempt": 1, "error_code": null, "at": "2026-11-19T15:11:43.910272+00:00"}, {"event": "retry", "attempt": 1, "error_code": "storage_unavailable", "at": "2026-11-19T15:11:43.913931+00:00"}, {"event": "processing", "attempt": 2, "error_code": null, "at": "2026-11-19T15:11:44.162093+00:00"}, {"event": "retry", "attempt": 2, "error_code": "storage_unavailable", "at": "2026-11-19T15:11:44.163784+00:00"}, {"event": "processing", "attempt": 3, "error_code": null, "at": "2026-11-19T15:11:44.672423+00:00"}, {"event": "retry", "attempt": 3, "error_code": "storage_unavailable", "at": "2026-11-19T15:11:44.674937+00:00"}, {"event": "processing", "attempt": 4, "error_code": null, "at": "2026-11-19T15:11:45.176057+00:00"}, {"event": "retry", "attempt": 4, "error_code": "storage_unavailable", "at": "2026-11-19T15:11:45.17712+00:00"}, {"event": "processing", "attempt": 5, "error_code": null, "at": "2026-11-19T15:11:45.941529+00:00"}, {"event": "failed", "attempt": 5, "error_code": "retry_exhausted", "at": "2026-11-19T15:11:45.944804+00:00"}, {"event": "operator_replay", "attempt": 5, "error_code": null, "at": "2026-11-19T15:11:47.029399+00:00"}, {"event": "processing", "attempt": 6, "error_code": null, "at": "2026-11-19T15:11:47.203814+00:00"}, {"event": "ready", "attempt": 6, "error_code": null, "at": "2026-11-19T15:11:47.601272+00:00"}], "ready_replay_rejected": true} +PASS: permanent_error {"job_id": "28022e30-0cd6-4df1-a41b-56905cf4feda", "attempt": 1, "error_code": "invalid_image"} +PASS: image_subprocess_timeout {"job_id": "c0f7e1b2-59a1-496e-a60a-8ed3fdb19f59", "timeout_seconds": 5, "task_memory_mb": 640, "recovered": true} +PASS: image_subprocess_memory {"job_id": "f1e4caee-86d7-4185-9787-92687c0650a3", "timeout_seconds": 5, "task_memory_mb": 640, "recovered": true} +PASS: image_semantics {"cases": [{"case": "small_alpha_on_white", "variants": {"thumbnail": {"size": [17, 9], "bytes": 635}, "crop": {"size": [128, 128], "bytes": 883}, "optimized": {"size": [17, 9], "bytes": 56}}}, {"case": "exif_orientation_6_and_floor", "variants": {"thumbnail": {"size": [160, 256], "bytes": 1270}, "crop": {"size": [128, 128], "bytes": 886}, "optimized": {"size": [201, 321], "bytes": 210}}}]} +PASS: staging_cleanup {"active_preserved": true, "orphan_removed": true, "ready_removed": true, "expired_failed_removed": true, "expired_replay_rejected": true} +PASS: broker_offline_outbox {"attachment_id": "28321e5b-8b43-471a-973d-9b55cbc07b39", "recovered": true} +/Users/vanek/Desktop/VNPlabs/VNPlab6/mtusi2026lab6/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:63012" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: public_persistence {"restart_proof": {"passed": true, "before": {"ea2b13aebb92a1ea2741de72078489e6b05ebcc41f1ca74a2604c0ad7d355983": {"service": "api", "started_at": "2026-11-19T15:10:30.841890222Z", "volumes": []}, "67eade4e281a4cfed3f55d87ab398097e12ee12155f82cbe19eb240a4e92b41a": {"service": "api", "started_at": "2026-11-19T15:10:30.809775805Z", "volumes": []}, "d675c92d195a1bb37db5860c3cf49a2a918eb3e98cfd9c855f8f89931138724b": {"service": "broker", "started_at": "2026-11-19T15:12:24.444290385Z", "volumes": ["lab5-check-c81d43a829_broker_data"]}, "f9b2e0b6e448dda6d845d9715d64e1216fc0f3f58c3e05ec22de30d305c2d3e6": {"service": "cleanup", "started_at": "2026-11-19T15:10:11.903050713Z", "volumes": []}, "e4abde6d7d326be2a1425d81667ab967dd655ce1eb39c1b0c1bd2f9956f3e91f": {"service": "grafana", "started_at": "2026-11-19T15:09:46.76721934Z", "volumes": ["lab5-check-c81d43a829_grafana_data"]}, "1e9b604ee9fe91572a4efd3b629a00819d86ab2b45ed1f04a1df66428cfe1920": {"service": "lb", "started_at": "2026-11-19T15:10:17.66400909Z", "volumes": []}, "487c1937cced6d3b86d6a67cae2b76fdfc2b534e6cf15b6cf2242b96b19c8af4": {"service": "loki", "started_at": "2026-11-19T15:09:46.088598506Z", "volumes": ["lab5-check-c81d43a829_loki_data"]}, "f31970f75546cfcf6c390fad538eb9a7ebc25d26c481caef09bf679a5af8e8d9": {"service": "otel-collector", "started_at": "2026-11-19T15:09:46.409972798Z", "volumes": []}, "82576012c7b7b5a5a352207679f4c327fe50cbd5c56337de33c16ec15b2b0501": {"service": "postgres", "started_at": "2026-11-19T15:09:46.08945334Z", "volumes": ["lab5-check-c81d43a829_postgres_data"]}, "6e305e5121dc3113f906c5689d1006091714d72b452acaabd07e3d366f1d13f6": {"service": "prometheus", "started_at": "2026-11-19T15:09:46.555234256Z", "volumes": ["lab5-check-c81d43a829_prometheus_data"]}, "fb35c350ab7fcff516b44e4f6933b1e293c896e6cee9bf29d06e71ba99a0f931": {"service": "publisher", "started_at": "2026-11-19T15:12:10.913909129Z", "volumes": []}, "8cdcd8bdac73e3bc827ef7bc5a77bad0696c1023cd330eb10206dcc6b29529cf": {"service": "s3", "started_at": "2026-11-19T15:10:08.720162878Z", "volumes": ["d6e81fdf6ec0a4bd30cb0461085c08461b951f8ae9c696222d04b21152ff6560"]}, "053f3a4b4e5cd9b28510f08e9bd8d07374c586af075cb62d5c5b49f62af5a6ac": {"service": "s3-storage", "started_at": "2026-11-19T15:09:46.086474548Z", "volumes": ["lab5-check-c81d43a829_s3_data"]}, "84473a25dbb67b410a0db8d8b0e029b84b264b3238da1961c7693f13d9e99634": {"service": "tempo", "started_at": "2026-11-19T15:09:46.086451215Z", "volumes": ["lab5-check-c81d43a829_tempo_data"]}, "7f3ff970cf57bedf3a8800a6bed5cc3a83f749dab07d9ac42425d14c7eafee80": {"service": "valkey", "started_at": "2026-11-19T15:09:46.08714634Z", "volumes": ["lab5-check-c81d43a829_valkey_data"]}, "521131dedb68623718c9d5d5c8dc8b0924dbfe6ec90acd3b21e08189a9a1a1e5": {"service": "worker", "started_at": "2026-11-19T15:12:21.892301301Z", "volumes": []}}, "after": {"ea2b13aebb92a1ea2741de72078489e6b05ebcc41f1ca74a2604c0ad7d355983": {"service": "api", "started_at": "2026-11-19T15:12:30.097473346Z", "volumes": []}, "67eade4e281a4cfed3f55d87ab398097e12ee12155f82cbe19eb240a4e92b41a": {"service": "api", "started_at": "2026-11-19T15:12:30.048159138Z", "volumes": []}, "d675c92d195a1bb37db5860c3cf49a2a918eb3e98cfd9c855f8f89931138724b": {"service": "broker", "started_at": "2026-11-19T15:12:28.168864679Z", "volumes": ["lab5-check-c81d43a829_broker_data"]}, "f9b2e0b6e448dda6d845d9715d64e1216fc0f3f58c3e05ec22de30d305c2d3e6": {"service": "cleanup", "started_at": "2026-11-19T15:12:35.75098096Z", "volumes": []}, "e4abde6d7d326be2a1425d81667ab967dd655ce1eb39c1b0c1bd2f9956f3e91f": {"service": "grafana", "started_at": "2026-11-19T15:12:28.016722095Z", "volumes": ["lab5-check-c81d43a829_grafana_data"]}, "1e9b604ee9fe91572a4efd3b629a00819d86ab2b45ed1f04a1df66428cfe1920": {"service": "lb", "started_at": "2026-11-19T15:12:27.799505637Z", "volumes": []}, "487c1937cced6d3b86d6a67cae2b76fdfc2b534e6cf15b6cf2242b96b19c8af4": {"service": "loki", "started_at": "2026-11-19T15:12:29.867573388Z", "volumes": ["lab5-check-c81d43a829_loki_data"]}, "f31970f75546cfcf6c390fad538eb9a7ebc25d26c481caef09bf679a5af8e8d9": {"service": "otel-collector", "started_at": "2026-11-19T15:12:29.704735929Z", "volumes": []}, "82576012c7b7b5a5a352207679f4c327fe50cbd5c56337de33c16ec15b2b0501": {"service": "postgres", "started_at": "2026-11-19T15:12:28.084774804Z", "volumes": ["lab5-check-c81d43a829_postgres_data"]}, "6e305e5121dc3113f906c5689d1006091714d72b452acaabd07e3d366f1d13f6": {"service": "prometheus", "started_at": "2026-11-19T15:12:28.052574179Z", "volumes": ["lab5-check-c81d43a829_prometheus_data"]}, "fb35c350ab7fcff516b44e4f6933b1e293c896e6cee9bf29d06e71ba99a0f931": {"service": "publisher", "started_at": "2026-11-19T15:12:35.770825668Z", "volumes": []}, "8cdcd8bdac73e3bc827ef7bc5a77bad0696c1023cd330eb10206dcc6b29529cf": {"service": "s3", "started_at": "2026-11-19T15:12:28.115603887Z", "volumes": ["d6e81fdf6ec0a4bd30cb0461085c08461b951f8ae9c696222d04b21152ff6560"]}, "053f3a4b4e5cd9b28510f08e9bd8d07374c586af075cb62d5c5b49f62af5a6ac": {"service": "s3-storage", "started_at": "2026-11-19T15:12:35.714312418Z", "volumes": ["lab5-check-c81d43a829_s3_data"]}, "84473a25dbb67b410a0db8d8b0e029b84b264b3238da1961c7693f13d9e99634": {"service": "tempo", "started_at": "2026-11-19T15:12:28.142091095Z", "volumes": ["lab5-check-c81d43a829_tempo_data"]}, "7f3ff970cf57bedf3a8800a6bed5cc3a83f749dab07d9ac42425d14c7eafee80": {"service": "valkey", "started_at": "2026-11-19T15:12:28.230646637Z", "volumes": ["lab5-check-c81d43a829_valkey_data"]}, "521131dedb68623718c9d5d5c8dc8b0924dbfe6ec90acd3b21e08189a9a1a1e5": {"service": "worker", "started_at": "2026-11-19T15:12:27.946355887Z", "volumes": []}}, "same_containers_and_volumes": true}} +/Users/vanek/Desktop/VNPlabs/VNPlab6/mtusi2026lab6/.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:63012" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 110, "failed_attempts": 1, "first_recovery_seconds": 2.561} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: public_api_failover {} +PASS: all lab5 grade-4 criteria completed +Removed isolated test stack: lab5-check-c81d43a829 diff --git a/evidence/lab6-promtool.txt b/evidence/lab6-promtool.txt new file mode 100644 index 0000000..e05b605 --- /dev/null +++ b/evidence/lab6-promtool.txt @@ -0,0 +1 @@ + SUCCESS diff --git a/evidence/lab6-restart.json b/evidence/lab6-restart.json new file mode 100644 index 0000000..abc3084 --- /dev/null +++ b/evidence/lab6-restart.json @@ -0,0 +1,206 @@ +{ + "utc": "2026-11-19T15:11:37.994748+00:00", + "command": ".venv/bin/python student-tests/verify_restart.py", + "passed": true, + "before": { + "166e3f521d4f8b2206766859d89df193d0ebc44fb7b8c2d597da65ebefaaf1af": { + "service": "api", + "started_at": "2026-11-19T15:08:38.453676544Z", + "volumes": [] + }, + "863415a50886df6498a2c5d137298288e403eb8f628fd93088a93174bb744d8c": { + "service": "api", + "started_at": "2026-11-19T15:08:38.611590211Z", + "volumes": [] + }, + "ea51e5f3e0e7921a34c8678a17da799c5f440f0423fc5cc0d0ebc88b6be109b8": { + "service": "broker", + "started_at": "2026-11-19T15:00:33.36494925Z", + "volumes": [ + "mtusi-lab6_broker_data" + ] + }, + "d24bddbf342d9bc0f3b43d8462e7d991d82e38c848d4ef762a8b0b1cbb5959ba": { + "service": "cleanup", + "started_at": "2026-11-19T15:08:38.446185961Z", + "volumes": [] + }, + "375a3083ca7919b26cdc327adf969072cd09d91f3dc664037b63fb890c91849a": { + "service": "grafana", + "started_at": "2026-11-19T15:00:33.781731Z", + "volumes": [ + "mtusi-lab6_grafana_data" + ] + }, + "f56ed57943f2889033e15612d17d633e0b7bb3a9cceb9cb2ab336524117e6ffa": { + "service": "lb", + "started_at": "2026-11-19T15:00:58.904707679Z", + "volumes": [] + }, + "e107e6ede3c3b2be8a6995ef242f614dc6f592cf230f10d7862685696991ae71": { + "service": "loki", + "started_at": "2026-11-19T15:10:28.625123846Z", + "volumes": [ + "mtusi-lab6_loki_data" + ] + }, + "6e21c63012d160a406e28a342d668de0be55107aa16228ed9d5dc15c34b447be": { + "service": "otel-collector", + "started_at": "2026-11-19T15:01:52.896095343Z", + "volumes": [] + }, + "1b3cdbc4689c6b855a435acab8112f86a3c9dc5e0207257c31e7aaa650635a0a": { + "service": "postgres", + "started_at": "2026-11-19T15:00:33.362963334Z", + "volumes": [ + "mtusi-lab6_postgres_data" + ] + }, + "d54da34ab176f240c2519bb6fcc46417b5ea3d27de86e2ed05321675f14e1e23": { + "service": "prometheus", + "started_at": "2026-11-19T15:00:33.65938775Z", + "volumes": [ + "mtusi-lab6_prometheus_data" + ] + }, + "0cd7cf668f46d355febf0dd912003289a65966692b21a759997e0a8d1c5d83eb": { + "service": "publisher", + "started_at": "2026-11-19T15:08:38.448713669Z", + "volumes": [] + }, + "b7b3fe9f4f328f57dc0df0a3af84d8426e431505ffd236da9d6cc143123380c9": { + "service": "s3", + "started_at": "2026-11-19T15:00:49.997656091Z", + "volumes": [ + "a6b780554a2ebb58edb3e2f7b77fe7ee705ce9543ef0dbe0c8d9c4d80bc31da2" + ] + }, + "1b69e90a9f7108e2e5d2017365e82fa30495c42f5f93809d42b2fdfcc1f0a9a0": { + "service": "s3-storage", + "started_at": "2026-11-19T15:00:33.3498635Z", + "volumes": [ + "mtusi-lab6_s3_data" + ] + }, + "7dc0833e289e6e708dff042076bf4753617ba5e61122a9a67dd61c37384f1c9b": { + "service": "tempo", + "started_at": "2026-11-19T15:08:38.454968128Z", + "volumes": [ + "mtusi-lab6_tempo_data" + ] + }, + "f4c736f41e428fb83e0effb49e8a56d9efe23ea6bd979a16fdcf072af0b342b0": { + "service": "valkey", + "started_at": "2026-11-19T15:00:33.364171167Z", + "volumes": [ + "mtusi-lab6_valkey_data" + ] + }, + "ef1c4fbd5187a8773b7b448cb5f0b0534e83bddb9b12a51f2ffcd7a326e5e729": { + "service": "worker", + "started_at": "2026-11-19T15:10:59.920254179Z", + "volumes": [] + } + }, + "after": { + "166e3f521d4f8b2206766859d89df193d0ebc44fb7b8c2d597da65ebefaaf1af": { + "service": "api", + "started_at": "2026-11-19T15:11:41.117466337Z", + "volumes": [] + }, + "863415a50886df6498a2c5d137298288e403eb8f628fd93088a93174bb744d8c": { + "service": "api", + "started_at": "2026-11-19T15:11:41.169749671Z", + "volumes": [] + }, + "ea51e5f3e0e7921a34c8678a17da799c5f440f0423fc5cc0d0ebc88b6be109b8": { + "service": "broker", + "started_at": "2026-11-19T15:11:40.171899503Z", + "volumes": [ + "mtusi-lab6_broker_data" + ] + }, + "d24bddbf342d9bc0f3b43d8462e7d991d82e38c848d4ef762a8b0b1cbb5959ba": { + "service": "cleanup", + "started_at": "2026-11-19T15:11:47.626797257Z", + "volumes": [] + }, + "375a3083ca7919b26cdc327adf969072cd09d91f3dc664037b63fb890c91849a": { + "service": "grafana", + "started_at": "2026-11-19T15:11:39.799567837Z", + "volumes": [ + "mtusi-lab6_grafana_data" + ] + }, + "f56ed57943f2889033e15612d17d633e0b7bb3a9cceb9cb2ab336524117e6ffa": { + "service": "lb", + "started_at": "2026-11-19T15:11:39.674464462Z", + "volumes": [] + }, + "e107e6ede3c3b2be8a6995ef242f614dc6f592cf230f10d7862685696991ae71": { + "service": "loki", + "started_at": "2026-11-19T15:11:40.282189378Z", + "volumes": [ + "mtusi-lab6_loki_data" + ] + }, + "6e21c63012d160a406e28a342d668de0be55107aa16228ed9d5dc15c34b447be": { + "service": "otel-collector", + "started_at": "2026-11-19T15:11:39.989740337Z", + "volumes": [] + }, + "1b3cdbc4689c6b855a435acab8112f86a3c9dc5e0207257c31e7aaa650635a0a": { + "service": "postgres", + "started_at": "2026-11-19T15:11:40.043447628Z", + "volumes": [ + "mtusi-lab6_postgres_data" + ] + }, + "d54da34ab176f240c2519bb6fcc46417b5ea3d27de86e2ed05321675f14e1e23": { + "service": "prometheus", + "started_at": "2026-11-19T15:11:39.859058378Z", + "volumes": [ + "mtusi-lab6_prometheus_data" + ] + }, + "0cd7cf668f46d355febf0dd912003289a65966692b21a759997e0a8d1c5d83eb": { + "service": "publisher", + "started_at": "2026-11-19T15:11:47.594100007Z", + "volumes": [] + }, + "b7b3fe9f4f328f57dc0df0a3af84d8426e431505ffd236da9d6cc143123380c9": { + "service": "s3", + "started_at": "2026-11-19T15:11:40.13670567Z", + "volumes": [ + "a6b780554a2ebb58edb3e2f7b77fe7ee705ce9543ef0dbe0c8d9c4d80bc31da2" + ] + }, + "1b69e90a9f7108e2e5d2017365e82fa30495c42f5f93809d42b2fdfcc1f0a9a0": { + "service": "s3-storage", + "started_at": "2026-11-19T15:11:47.65993734Z", + "volumes": [ + "mtusi-lab6_s3_data" + ] + }, + "7dc0833e289e6e708dff042076bf4753617ba5e61122a9a67dd61c37384f1c9b": { + "service": "tempo", + "started_at": "2026-11-19T15:11:40.243825753Z", + "volumes": [ + "mtusi-lab6_tempo_data" + ] + }, + "f4c736f41e428fb83e0effb49e8a56d9efe23ea6bd979a16fdcf072af0b342b0": { + "service": "valkey", + "started_at": "2026-11-19T15:11:40.087174462Z", + "volumes": [ + "mtusi-lab6_valkey_data" + ] + }, + "ef1c4fbd5187a8773b7b448cb5f0b0534e83bddb9b12a51f2ffcd7a326e5e729": { + "service": "worker", + "started_at": "2026-11-19T15:11:40.21509467Z", + "volumes": [] + } + }, + "same_containers_and_volumes": true +} diff --git a/evidence/lab6-restart.txt b/evidence/lab6-restart.txt new file mode 100644 index 0000000..32dc713 --- /dev/null +++ b/evidence/lab6-restart.txt @@ -0,0 +1,4 @@ +/Users/vanek/Desktop/VNPlabs/VNPlab6/mtusi2026lab6/.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +PASS: all persistent services actually restarted; container IDs and volumes preserved diff --git a/evidence/lab6-runtime.json b/evidence/lab6-runtime.json new file mode 100644 index 0000000..7778791 --- /dev/null +++ b/evidence/lab6-runtime.json @@ -0,0 +1,322 @@ +{ + "passed": true, + "services": [ + { + "service": "api", + "container_id": "33daefd247cfa9bdb99c0e9d10913c5305de5db293431162467a9fb0f55f3e4c", + "image": "mtusi-messenger:lab6", + "image_id": "sha256:a7932cb2a948db89797bf90f560ecf0046cac576c7fff97412aa55d8c4446a62", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 1000000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [] + }, + { + "service": "api", + "container_id": "2a50b728ce1094a536622ed51528f1e359b5bbd51685bb3b368dfb9327a2f9d8", + "image": "mtusi-messenger:lab6", + "image_id": "sha256:a7932cb2a948db89797bf90f560ecf0046cac576c7fff97412aa55d8c4446a62", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 1000000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [] + }, + { + "service": "broker", + "container_id": "ea51e5f3e0e7921a34c8678a17da799c5f440f0423fc5cc0d0ebc88b6be109b8", + "image": "valkey/valkey:8.1.3", + "image_id": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 268435456, + "nano_cpus": 500000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_broker_data" + ] + }, + { + "service": "cleanup", + "container_id": "e3328a92cd7b264a34d3ba2863ebd509ac105e68d507ce39062181cbe34e91bc", + "image": "mtusi-messenger:lab6", + "image_id": "sha256:a7932cb2a948db89797bf90f560ecf0046cac576c7fff97412aa55d8c4446a62", + "state": "running", + "health": null, + "memory_limit_bytes": 402653184, + "nano_cpus": 1000000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [] + }, + { + "service": "grafana", + "container_id": "9d373574ca2ee05f2e55ffc54a58266bfe13ac1c12d46c5275355a716fa8bd70", + "image": "grafana/grafana:12.0.2", + "image_id": "sha256:b5b59bfc7561634c2d7b136c4543d702ebcc94a3da477f21ff26f89ffd4214fa", + "state": "running", + "health": null, + "memory_limit_bytes": 402653184, + "nano_cpus": 1000000000, + "oom_killed": false, + "published_ports": { + "3000/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "3000" + } + ] + }, + "volumes": [ + "mtusi-lab6_grafana_data" + ] + }, + { + "service": "lb", + "container_id": "f56ed57943f2889033e15612d17d633e0b7bb3a9cceb9cb2ab336524117e6ffa", + "image": "haproxy:3.2.6-alpine", + "image_id": "sha256:3a819392f5a6af2a9203d1c63ff0feba74f0c940623a558511be06d7d85e361c", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 67108864, + "nano_cpus": 500000000, + "oom_killed": false, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [] + }, + { + "service": "loki", + "container_id": "e107e6ede3c3b2be8a6995ef242f614dc6f592cf230f10d7862685696991ae71", + "image": "grafana/loki:3.5.1", + "image_id": "sha256:a74594532eec4cc313401beedc4dd2708c43674c032084b1aeb87c14a5be1745", + "state": "running", + "health": null, + "memory_limit_bytes": 402653184, + "nano_cpus": 1000000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_loki_data" + ] + }, + { + "service": "migrate", + "container_id": "a01249e7317b529181758de5244cd14133cc7b39ba5077ee3d844d240ca98cb2", + "image": "mtusi-messenger:lab6", + "image_id": "sha256:0189284e9808367735919f731c1169eb64ba34743d8f5dcaebf73a9f6788902c", + "state": "exited", + "health": null, + "memory_limit_bytes": 402653184, + "nano_cpus": 1000000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [] + }, + { + "service": "otel-collector", + "container_id": "6e21c63012d160a406e28a342d668de0be55107aa16228ed9d5dc15c34b447be", + "image": "otel/opentelemetry-collector-contrib:0.123.0", + "image_id": "sha256:e39311df1f3d941923c00da79ac7ba6269124a870ee87e3c3ad24d60f8aee4d2", + "state": "running", + "health": null, + "memory_limit_bytes": 268435456, + "nano_cpus": 1000000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [] + }, + { + "service": "postgres", + "container_id": "1b3cdbc4689c6b855a435acab8112f86a3c9dc5e0207257c31e7aaa650635a0a", + "image": "postgres:17.6-bookworm", + "image_id": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_postgres_data" + ] + }, + { + "service": "prometheus", + "container_id": "d54da34ab176f240c2519bb6fcc46417b5ea3d27de86e2ed05321675f14e1e23", + "image": "prom/prometheus:v3.4.1", + "image_id": "sha256:9abc6cf6aea7710d163dbb28d8eeb7dc5baef01e38fa4cd146a406dd9f07f70d", + "state": "running", + "health": null, + "memory_limit_bytes": 402653184, + "nano_cpus": 1000000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_prometheus_data" + ] + }, + { + "service": "publisher", + "container_id": "02738dc58bbbadc9dc35910593e6a75bcb1794821b38f892d22f05c0f9df7772", + "image": "mtusi-messenger:lab6", + "image_id": "sha256:a7932cb2a948db89797bf90f560ecf0046cac576c7fff97412aa55d8c4446a62", + "state": "running", + "health": null, + "memory_limit_bytes": 402653184, + "nano_cpus": 1000000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [] + }, + { + "service": "s3", + "container_id": "b7b3fe9f4f328f57dc0df0a3af84d8426e431505ffd236da9d6cc143123380c9", + "image": "chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "image_id": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 268435456, + "nano_cpus": 500000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [ + "a6b780554a2ebb58edb3e2f7b77fe7ee705ce9543ef0dbe0c8d9c4d80bc31da2" + ] + }, + { + "service": "s3-bootstrap", + "container_id": "4c9c5e1d677c45da83a1651c20809f58457588843791e01d5b6e22ab486c9467", + "image": "mtusi-messenger:lab6", + "image_id": "sha256:0189284e9808367735919f731c1169eb64ba34743d8f5dcaebf73a9f6788902c", + "state": "exited", + "health": null, + "memory_limit_bytes": 134217728, + "nano_cpus": 500000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [] + }, + { + "service": "s3-storage", + "container_id": "1b69e90a9f7108e2e5d2017365e82fa30495c42f5f93809d42b2fdfcc1f0a9a0", + "image": "chrislusf/seaweedfs:3.85@sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "image_id": "sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_s3_data" + ] + }, + { + "service": "tempo", + "container_id": "7dc0833e289e6e708dff042076bf4753617ba5e61122a9a67dd61c37384f1c9b", + "image": "grafana/tempo:2.8.1", + "image_id": "sha256:bc9245fe3da4e63dc4c6862d9c2dad9bcd8be13d0ba4f7705fa6acda4c904d0e", + "state": "running", + "health": null, + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_tempo_data" + ] + }, + { + "service": "valkey", + "container_id": "f4c736f41e428fb83e0effb49e8a56d9efe23ea6bd979a16fdcf072af0b342b0", + "image": "valkey/valkey:8.1.3", + "image_id": "sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 500000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [ + "mtusi-lab6_valkey_data" + ] + }, + { + "service": "worker", + "container_id": "168b98028f54233914e74d5a5e4e0f75b68968230fce783e9835e768e1450813", + "image": "mtusi-messenger:lab6", + "image_id": "sha256:a7932cb2a948db89797bf90f560ecf0046cac576c7fff97412aa55d8c4446a62", + "state": "running", + "health": null, + "memory_limit_bytes": 1610612736, + "nano_cpus": 2000000000, + "oom_killed": false, + "published_ports": {}, + "volumes": [] + } + ], + "grafana_version": "12.0.2", + "datasources": [ + { + "uid": "loki", + "name": "Loki", + "type": "loki", + "url": "http://loki:3100", + "jsonData": { + "derivedFields": [ + { + "datasourceUid": "tempo", + "matcherRegex": "\"trace_id\": \"([a-f0-9]{32})\"", + "name": "TraceID", + "url": "${__value.raw}" + } + ] + } + }, + { + "uid": "prometheus", + "name": "Prometheus", + "type": "prometheus", + "url": "http://prometheus:9090", + "jsonData": { + "timeInterval": "5s" + } + }, + { + "uid": "tempo", + "name": "Tempo", + "type": "tempo", + "url": "http://tempo:3200", + "jsonData": { + "nodeGraph": { + "enabled": true + }, + "tracesToLogs": { + "datasourceUid": "loki", + "filterByTraceID": true, + "spanEndTimeShift": "1m", + "spanStartTimeShift": "-1m", + "tags": [ + { + "key": "service.name", + "value": "service_name" + } + ] + } + } + } + ] +} diff --git a/evidence/lab6-sdk-buffers.json b/evidence/lab6-sdk-buffers.json new file mode 100644 index 0000000..adf4e96 --- /dev/null +++ b/evidence/lab6-sdk-buffers.json @@ -0,0 +1,666 @@ +{ + "passed": true, + "metrics": [ + { + "metric": { + "__name__": "otel_sdk_processor_log_queue_capacity", + "deployment_environment_name": "lab", + "exported_instance": "02738dc58bbb:1", + "exported_job": "messenger-publisher", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "02738dc58bbb:1", + "service_name": "messenger-publisher", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "256" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_queue_capacity", + "deployment_environment_name": "lab", + "exported_instance": "168b98028f54:8", + "exported_job": "messenger-worker", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "168b98028f54:8", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "256" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_queue_capacity", + "deployment_environment_name": "lab", + "exported_instance": "168b98028f54:9", + "exported_job": "messenger-worker", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "168b98028f54:9", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "256" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_queue_capacity", + "deployment_environment_name": "lab", + "exported_instance": "2a50b728ce10:1", + "exported_job": "messenger-api", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "2a50b728ce10:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "256" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_queue_capacity", + "deployment_environment_name": "lab", + "exported_instance": "33daefd247cf:1", + "exported_job": "messenger-api", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "33daefd247cf:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "256" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_queue_size", + "deployment_environment_name": "lab", + "exported_instance": "02738dc58bbb:1", + "exported_job": "messenger-publisher", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "02738dc58bbb:1", + "service_name": "messenger-publisher", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "0" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_queue_size", + "deployment_environment_name": "lab", + "exported_instance": "168b98028f54:8", + "exported_job": "messenger-worker", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "168b98028f54:8", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "0" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_queue_size", + "deployment_environment_name": "lab", + "exported_instance": "168b98028f54:9", + "exported_job": "messenger-worker", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "168b98028f54:9", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "0" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_queue_size", + "deployment_environment_name": "lab", + "exported_instance": "2a50b728ce10:1", + "exported_job": "messenger-api", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "2a50b728ce10:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "0" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_queue_size", + "deployment_environment_name": "lab", + "exported_instance": "33daefd247cf:1", + "exported_job": "messenger-api", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "33daefd247cf:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "0" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_queue_capacity", + "deployment_environment_name": "lab", + "exported_instance": "02738dc58bbb:1", + "exported_job": "messenger-publisher", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "02738dc58bbb:1", + "service_name": "messenger-publisher", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "256" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_queue_capacity", + "deployment_environment_name": "lab", + "exported_instance": "168b98028f54:8", + "exported_job": "messenger-worker", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "168b98028f54:8", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "256" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_queue_capacity", + "deployment_environment_name": "lab", + "exported_instance": "168b98028f54:9", + "exported_job": "messenger-worker", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "168b98028f54:9", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "256" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_queue_capacity", + "deployment_environment_name": "lab", + "exported_instance": "2a50b728ce10:1", + "exported_job": "messenger-api", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "2a50b728ce10:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "256" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_queue_capacity", + "deployment_environment_name": "lab", + "exported_instance": "33daefd247cf:1", + "exported_job": "messenger-api", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "33daefd247cf:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "256" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_queue_size", + "deployment_environment_name": "lab", + "exported_instance": "02738dc58bbb:1", + "exported_job": "messenger-publisher", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "02738dc58bbb:1", + "service_name": "messenger-publisher", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "0" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_queue_size", + "deployment_environment_name": "lab", + "exported_instance": "168b98028f54:8", + "exported_job": "messenger-worker", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "168b98028f54:8", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "0" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_queue_size", + "deployment_environment_name": "lab", + "exported_instance": "168b98028f54:9", + "exported_job": "messenger-worker", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "168b98028f54:9", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "0" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_queue_size", + "deployment_environment_name": "lab", + "exported_instance": "2a50b728ce10:1", + "exported_job": "messenger-api", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "2a50b728ce10:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "0" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_queue_size", + "deployment_environment_name": "lab", + "exported_instance": "33daefd247cf:1", + "exported_job": "messenger-api", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "33daefd247cf:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "0" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_processed_total", + "deployment_environment_name": "lab", + "exported_instance": "02738dc58bbb:1", + "exported_job": "messenger-publisher", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "02738dc58bbb:1", + "service_name": "messenger-publisher", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "318" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_processed_total", + "deployment_environment_name": "lab", + "exported_instance": "168b98028f54:9", + "exported_job": "messenger-worker", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "168b98028f54:9", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "66" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_processed_total", + "deployment_environment_name": "lab", + "exported_instance": "2a50b728ce10:1", + "exported_job": "messenger-api", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "2a50b728ce10:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "1296" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_processed_total", + "deployment_environment_name": "lab", + "exported_instance": "33daefd247cf:1", + "exported_job": "messenger-api", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "33daefd247cf:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "1289" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_processed_total", + "deployment_environment_name": "lab", + "exported_instance": "02738dc58bbb:1", + "exported_job": "messenger-publisher", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "02738dc58bbb:1", + "service_name": "messenger-publisher", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "318" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_processed_total", + "deployment_environment_name": "lab", + "exported_instance": "168b98028f54:9", + "exported_job": "messenger-worker", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "168b98028f54:9", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "1320" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_processed_total", + "deployment_environment_name": "lab", + "exported_instance": "2a50b728ce10:1", + "exported_job": "messenger-api", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "2a50b728ce10:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "13025" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_processed_total", + "deployment_environment_name": "lab", + "exported_instance": "33daefd247cf:1", + "exported_job": "messenger-api", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "33daefd247cf:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "13012" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_log_processed_total", + "deployment_environment_name": "lab", + "exported_instance": "168b98028f54:8", + "exported_job": "messenger-worker", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_log_processor/0", + "otel_component_type": "batching_log_processor", + "service_instance_id": "168b98028f54:8", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "54" + ] + }, + { + "metric": { + "__name__": "otel_sdk_processor_span_processed_total", + "deployment_environment_name": "lab", + "exported_instance": "168b98028f54:8", + "exported_job": "messenger-worker", + "instance": "otel-collector:9464", + "job": "messenger", + "otel_component_name": "batching_span_processor/0", + "otel_component_type": "batching_span_processor", + "service_instance_id": "168b98028f54:8", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1" + }, + "value": [ + 1795102653.838, + "1080" + ] + } + ], + "per_process_queue_capacity": 256 +} diff --git a/evidence/lab6-slo-research.json b/evidence/lab6-slo-research.json new file mode 100644 index 0000000..adc3266 --- /dev/null +++ b/evidence/lab6-slo-research.json @@ -0,0 +1,20389 @@ +{ + "utc": "2026-11-19T15:26:14.987289+00:00", + "hypothesis": "Multiwindow burn alerts ignore a 4s spike, detect sustained DB/worker degradation and resolve after recovery.", + "profile": { + "message_arrival_per_second": 2, + "image_arrival_per_second": 0.5, + "max_inflight": 16, + "sampling": 1.0, + "retries": 0, + "slo_window_seconds": 3600, + "api_slo": 0.99, + "ready_slo": 0.95, + "ready_deadline_seconds": 30 + }, + "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", + "db_diagnosis": { + "trace_id": "f124d93f84dd25c5af978e78e2028646", + "request_id": "3c5e5b14-5825-41b8-a227-93cba420a532", + "slow_error_spans": [ + { + "name": "postgresql.SELECT", + "status": { + "code": "STATUS_CODE_ERROR" + }, + "duration_ms": 2000.115335, + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "_WaitTimeout" + } + } + ] + }, + { + "name": "postgresql.transaction", + "status": { + "code": "STATUS_CODE_ERROR" + }, + "duration_ms": 2001.169168, + "attributes": [ + { + "key": "error.type", + "value": { + "stringValue": "_WaitTimeout" + } + } + ] + }, + { + "name": "POST /api/v1/chats/{chat_id}/messages", + "status": { + "code": "STATUS_CODE_ERROR" + }, + "duration_ms": 2003.784959, + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/messages" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ] + } + ] + }, + "short_spike_observed_error_ratio_peak": 0.06373490138007629, + "passed": true, + "false_positives": { + "baseline": 0, + "burn_alerts_after_short_spike": 0, + "single_window_alerts_after_short_spike": [ + "APIHighErrorRate" + ] + }, + "start": 1795101974.987257, + "end": 1795102594.377363, + "duration_seconds": 619.3901059627533, + "phases": [ + { + "name": "baseline", + "start": 1795101975.364327, + "planned_seconds": 120, + "active_at": 1795101975.364415, + "end": 1795102095.3712242 + }, + { + "name": "short_db_spike", + "start": 1795102095.371294, + "planned_seconds": 4, + "active_at": 1795102095.527463, + "end": 1795102099.381341 + }, + { + "name": "after_short_spike", + "start": 1795102099.381383, + "planned_seconds": 60, + "active_at": 1795102099.381435, + "end": 1795102159.386289 + }, + { + "name": "sustained_db_lock", + "start": 1795102159.386304, + "planned_seconds": 90, + "active_at": 1795102159.552147, + "end": 1795102249.3866298 + }, + { + "name": "after_db_recovery", + "start": 1795102252.4302201, + "planned_seconds": 100, + "active_at": 1795102252.430262, + "end": 1795102352.43217 + }, + { + "name": "worker_stopped", + "start": 1795102352.4322171, + "planned_seconds": 120, + "active_at": 1795102352.766314, + "end": 1795102472.4357738 + }, + { + "name": "after_worker_recovery", + "start": 1795102472.435835, + "planned_seconds": 120, + "active_at": 1795102474.322918, + "end": 1795102592.4362948 + } + ], + "samples": [ + { + "utc": 1795101975.370796, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795101975.372, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795101975.375, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795101975.377, + "NaN" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795101975.38, + "0" + ] + } + ] + }, + { + "utc": 1795101980.402409, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795101980.405, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795101980.409, + "0.479455" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795101980.412, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795101980.414, + "0" + ] + } + ] + }, + { + "utc": 1795101985.431202, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795101985.433, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795101985.436, + "0.191489" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795101985.439, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795101985.442, + "0" + ] + } + ] + }, + { + "utc": 1795101990.454839, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795101990.457, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795101990.46, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795101990.464, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795101990.468, + "0" + ] + } + ] + }, + { + "utc": 1795101995.483205, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795101995.484, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795101995.487, + "0.43069" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795101995.489, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795101995.491, + "0" + ] + } + ] + }, + { + "utc": 1795102000.5113459, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102000.513, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102000.518, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102000.52, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102000.523, + "0" + ] + } + ] + }, + { + "utc": 1795102005.536093, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102005.536, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102005.539, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102005.541, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102005.543, + "0" + ] + } + ] + }, + { + "utc": 1795102010.5618382, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102010.567, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102010.573, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102010.578, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102010.582, + "0" + ] + } + ] + }, + { + "utc": 1795102015.5919712, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102015.592, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102015.597, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102015.601, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102015.605, + "0" + ] + } + ] + }, + { + "utc": 1795102020.6269178, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102020.63, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102020.637, + "0.051709" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102020.642, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102020.647, + "0" + ] + } + ] + }, + { + "utc": 1795102025.65963, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102025.659, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102025.661, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102025.663, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102025.665, + "0" + ] + } + ] + }, + { + "utc": 1795102030.682538, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102030.687, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102030.692, + "0.286549" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102030.697, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102030.702, + "0" + ] + } + ] + }, + { + "utc": 1795102035.717729, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102035.718, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102035.719, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102035.721, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102035.723, + "0" + ] + } + ] + }, + { + "utc": 1795102040.742848, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102040.747, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102040.753, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102040.757, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102040.762, + "0" + ] + } + ] + }, + { + "utc": 1795102045.7736802, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102045.774, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102045.775, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102045.777, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102045.779, + "0" + ] + } + ] + }, + { + "utc": 1795102050.793431, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102050.797, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102050.804, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102050.811, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102050.814, + "0" + ] + } + ] + }, + { + "utc": 1795102055.8255968, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102055.825, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102055.827, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102055.829, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102055.832, + "0" + ] + } + ] + }, + { + "utc": 1795102060.851578, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102060.854, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102060.859, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102060.865, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102060.869, + "0" + ] + } + ] + }, + { + "utc": 1795102065.876627, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102065.876, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102065.877, + "0.148366" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102065.879, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102065.88, + "0" + ] + } + ] + }, + { + "utc": 1795102070.899188, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102070.9, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102070.904, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102070.907, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102070.911, + "0" + ] + } + ] + }, + { + "utc": 1795102075.928641, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102075.928, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102075.93, + "0.427642" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102075.932, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102075.934, + "0" + ] + } + ] + }, + { + "utc": 1795102080.9533079, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102080.956, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102080.961, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102080.967, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102080.971, + "0" + ] + } + ] + }, + { + "utc": 1795102085.98634, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102085.987, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102085.99, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102085.992, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102085.994, + "0" + ] + } + ] + }, + { + "utc": 1795102091.006878, + "phase": "baseline", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102091.009, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102091.012, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102091.015, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102091.017, + "0" + ] + } + ] + }, + { + "utc": 1795102095.5298612, + "phase": "short_db_spike", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102095.53, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102095.532, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102095.535, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102095.537, + "0" + ] + } + ] + }, + { + "utc": 1795102099.3964329, + "phase": "after_short_spike", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102099.399, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102099.402, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102099.406, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102099.409, + "0" + ] + } + ] + }, + { + "utc": 1795102104.420933, + "phase": "after_short_spike", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102104.424, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102104.43, + "0.090217" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102104.433, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102104.436, + "0" + ] + } + ] + }, + { + "utc": 1795102109.448571, + "phase": "after_short_spike", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102109.45, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102109.452, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102109.454, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102109.457, + "0" + ] + } + ] + }, + { + "utc": 1795102114.471339, + "phase": "after_short_spike", + "alerts": [ + { + "name": "APIHighErrorRate", + "state": "pending", + "activeAt": "2026-11-19T15:28:29.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102114.473, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102114.476, + "0.30598" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102114.479, + "0.06373490138007627" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102114.484, + "0.013892844416217593" + ] + } + ] + }, + { + "utc": 1795102119.4955661, + "phase": "after_short_spike", + "alerts": [ + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:28:29.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102119.496, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102119.499, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102119.501, + "0.06272240485308571" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102119.503, + "0.01384380420169672" + ] + } + ] + }, + { + "utc": 1795102124.508966, + "phase": "after_short_spike", + "alerts": [ + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:28:29.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102124.51, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102124.512, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102124.514, + "0.06373490138007629" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102124.516, + "0.013892386273567859" + ] + } + ] + }, + { + "utc": 1795102129.538003, + "phase": "after_short_spike", + "alerts": [ + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:28:29.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102129.541, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102129.547, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102129.552, + "0.06272240485308571" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102129.557, + "0.013844146632578701" + ] + } + ] + }, + { + "utc": 1795102134.570018, + "phase": "after_short_spike", + "alerts": [ + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:28:29.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102134.57, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102134.572, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102134.574, + "0.06373490138007627" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102134.575, + "0.013892500809270198" + ] + } + ] + }, + { + "utc": 1795102139.585644, + "phase": "after_short_spike", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102139.586, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102139.588, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102139.591, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102139.593, + "0.01384460320671808" + ] + } + ] + }, + { + "utc": 1795102144.607593, + "phase": "after_short_spike", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102144.608, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102144.611, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102144.613, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102144.615, + "0.013892844416217595" + ] + } + ] + }, + { + "utc": 1795102149.6338828, + "phase": "after_short_spike", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102149.636, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102149.64, + "0.242537" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102149.643, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102149.646, + "0.01384460320671808" + ] + } + ] + }, + { + "utc": 1795102154.6610858, + "phase": "after_short_spike", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102154.663, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102154.667, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102154.668, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102154.67, + "0.013892844416217595" + ] + } + ] + }, + { + "utc": 1795102159.554812, + "phase": "sustained_db_lock", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102159.555, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102159.557, + "0.473372" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102159.559, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102159.561, + "0.01384460320671808" + ] + } + ] + }, + { + "utc": 1795102164.571063, + "phase": "sustained_db_lock", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102164.573, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102164.575, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102164.578, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102164.581, + "0.013892844416217595" + ] + } + ] + }, + { + "utc": 1795102169.590027, + "phase": "sustained_db_lock", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102169.592, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102169.594, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102169.597, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102169.599, + "0.013843804201696721" + ] + } + ] + }, + { + "utc": 1795102174.6142159, + "phase": "sustained_db_lock", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102174.614, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102174.617, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102174.619, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102174.621, + "0.013990342485717481" + ] + } + ] + }, + { + "utc": 1795102179.6360161, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIHighErrorRate", + "state": "pending", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102179.64, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102179.645, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102179.649, + "0.13559322033898305" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102179.652, + "0.042120294900170975" + ] + } + ] + }, + { + "utc": 1795102184.665347, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "pending", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102184.666, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102184.668, + "0.116103" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102184.669, + "0.3103448275862069" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102184.672, + "0.07750664301250751" + ] + } + ] + }, + { + "utc": 1795102189.682394, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "pending", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102189.683, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102189.685, + "0.116103" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102189.686, + "0.47457627118644063" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102189.688, + "0.11235016799594966" + ] + } + ] + }, + { + "utc": 1795102194.700725, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "pending", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "pending", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102194.701, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102194.705, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102194.709, + "0.6551724137931034" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102194.714, + "0.14800105492317514" + ] + } + ] + }, + { + "utc": 1795102199.7329679, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "pending", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102199.733, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102199.736, + "0.372153" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102199.738, + "0.7868852459016393" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102199.74, + "0.18261468409231948" + ] + } + ] + }, + { + "utc": 1795102204.773538, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "pending", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102204.775, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102204.777, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102204.782, + "0.7936507936507936" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102204.784, + "0.25290872236114675" + ] + } + ] + }, + { + "utc": 1795102209.790663, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "pending", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102209.792, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102209.794, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102209.796, + "0.8064516129032259" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102209.798, + "0.2890866189596236" + ] + } + ] + }, + { + "utc": 1795102214.805221, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "pending", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102214.808, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102214.813, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102214.816, + "0.7936507936507936" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102214.819, + "0.3232348060339257" + ] + } + ] + }, + { + "utc": 1795102219.8331552, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102219.834, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102219.835, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102219.837, + "0.8064516129032258" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102219.839, + "0.35967781534806287" + ] + } + ] + }, + { + "utc": 1795102224.855082, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102224.86, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102224.866, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102224.873, + "0.7936507936507936" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102224.878, + "0.3802816901408451" + ] + } + ] + }, + { + "utc": 1795102229.889604, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102229.89, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102229.893, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102229.896, + "0.8064516129032259" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102229.898, + "0.41696113074204955" + ] + } + ] + }, + { + "utc": 1795102234.9101388, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102234.912, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102234.916, + "0.236601" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102234.919, + "0.7936507936507936" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102234.922, + "0.45070422535211263" + ] + } + ] + }, + { + "utc": 1795102239.928961, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102239.93, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102239.931, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102239.933, + "0.8064516129032259" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102239.935, + "0.4876325088339223" + ] + } + ] + }, + { + "utc": 1795102244.956892, + "phase": "sustained_db_lock", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102244.96, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102244.964, + "0.510616" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102244.967, + "0.7936507936507936" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102244.97, + "0.5211267605633804" + ] + } + ] + }, + { + "utc": 1795102252.433323, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102252.434, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102252.436, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102252.438, + "0.8064516129032258" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102252.44, + "0.558303886925795" + ] + } + ] + }, + { + "utc": 1795102257.453194, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102257.456, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102257.459, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102257.464, + "0.7936507936507936" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102257.467, + "0.5915492957746479" + ] + } + ] + }, + { + "utc": 1795102262.4737978, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102262.474, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102262.477, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102262.479, + "0.7272727272727273" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102262.483, + "0.6132404181184669" + ] + } + ] + }, + { + "utc": 1795102267.491519, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102267.493, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102267.495, + "0.076095" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102267.496, + "0.5671641791044776" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102267.498, + "0.6111111111111112" + ] + } + ] + }, + { + "utc": 1795102272.51331, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102272.517, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102272.522, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102272.527, + "0.42424242424242425" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102272.533, + "0.613240418118467" + ] + } + ] + }, + { + "utc": 1795102277.538638, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102277.539, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102277.541, + "0.313001" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102277.543, + "0.26865671641791045" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102277.544, + "0.6111111111111112" + ] + } + ] + }, + { + "utc": 1795102282.556424, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilityFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:39.758725496Z" + }, + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + }, + { + "name": "APIHighErrorRate", + "state": "firing", + "activeAt": "2026-11-19T15:29:34.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102282.559, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102282.561, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102282.564, + "0.12121212121212122" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102282.567, + "0.6132404181184669" + ] + } + ] + }, + { + "utc": 1795102287.575625, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102287.578, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102287.582, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102287.585, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102287.587, + "0.6068965517241379" + ] + } + ] + }, + { + "utc": 1795102292.597072, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102292.597, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102292.6, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102292.603, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102292.604, + "0.577319587628866" + ] + } + ] + }, + { + "utc": 1795102297.6169229, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102297.618, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102297.621, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102297.624, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102297.627, + "0.5392491467576792" + ] + } + ] + }, + { + "utc": 1795102302.6381881, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102302.639, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102302.642, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102302.645, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102302.649, + "0.5068493150684932" + ] + } + ] + }, + { + "utc": 1795102307.656308, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102307.656, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102307.658, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102307.66, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102307.662, + "0.4709897610921501" + ] + } + ] + }, + { + "utc": 1795102312.68056, + "phase": "after_db_recovery", + "alerts": [ + { + "name": "APIAvailabilitySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:29:49.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102312.685, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102312.691, + "0.083168" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102312.696, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102312.699, + "0.4383561643835617" + ] + } + ] + }, + { + "utc": 1795102317.7049978, + "phase": "after_db_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102317.705, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102317.708, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102317.709, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102317.711, + "0.40273037542662116" + ] + } + ] + }, + { + "utc": 1795102322.722754, + "phase": "after_db_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102322.724, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102322.727, + "0.271703" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102322.73, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102322.734, + "0.3698630136986301" + ] + } + ] + }, + { + "utc": 1795102327.739033, + "phase": "after_db_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102327.74, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102327.743, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102327.745, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102327.747, + "0.33447098976109213" + ] + } + ] + }, + { + "utc": 1795102332.766411, + "phase": "after_db_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102332.771, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102332.776, + "0.493807" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102332.781, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102332.787, + "0.30136986301369867" + ] + } + ] + }, + { + "utc": 1795102337.798569, + "phase": "after_db_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102337.799, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102337.802, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102337.806, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102337.808, + "0.26621160409556316" + ] + } + ] + }, + { + "utc": 1795102342.8212109, + "phase": "after_db_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102342.824, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102342.83, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102342.836, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102342.841, + "0.23287671232876714" + ] + } + ] + }, + { + "utc": 1795102347.84972, + "phase": "after_db_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102347.85, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102347.852, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102347.853, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102347.855, + "0.1979522184300341" + ] + } + ] + }, + { + "utc": 1795102352.769394, + "phase": "worker_stopped", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102352.77, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102352.773, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102352.775, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102352.777, + "0.1643835616438356" + ] + } + ] + }, + { + "utc": 1795102357.79069, + "phase": "worker_stopped", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102357.792, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102357.795, + "0.03898" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102357.797, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102357.801, + "0.12969283276450513" + ] + } + ] + }, + { + "utc": 1795102362.8097558, + "phase": "worker_stopped", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102362.81, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102362.814, + "1.14508" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102362.816, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102362.82, + "0.0958904109589041" + ] + } + ] + }, + { + "utc": 1795102367.830467, + "phase": "worker_stopped", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102367.83, + "4" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102367.832, + "6.249117" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102367.834, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102367.835, + "0.06143344709897611" + ] + } + ] + }, + { + "utc": 1795102372.849184, + "phase": "worker_stopped", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102372.852, + "6" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102372.857, + "11.382691" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102372.863, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102372.868, + "0.0273972602739726" + ] + } + ] + }, + { + "utc": 1795102377.883855, + "phase": "worker_stopped", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102377.887, + "9" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102377.891, + "16.507015" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102377.896, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102377.899, + "0" + ] + } + ] + }, + { + "utc": 1795102382.904703, + "phase": "worker_stopped", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102382.905, + "11" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102382.908, + "21.648112" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102382.91, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102382.912, + "0" + ] + } + ] + }, + { + "utc": 1795102387.923724, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentQueueLag", + "state": "pending", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102387.926, + "14" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102387.93, + "26.774682" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102387.934, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102387.938, + "0" + ] + } + ] + }, + { + "utc": 1795102392.9494662, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentQueueLag", + "state": "pending", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102392.952, + "16" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102392.957, + "31.875267" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102392.962, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102392.965, + "0" + ] + } + ] + }, + { + "utc": 1795102397.975321, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102397.977, + "19" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102397.98, + "36.998759" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102397.983, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102397.985, + "0" + ] + } + ] + }, + { + "utc": 1795102402.996616, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102403.003, + "22" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102403.01, + "42.124386" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102403.016, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102403.02, + "0" + ] + } + ] + }, + { + "utc": 1795102408.036837, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102408.04, + "24" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102408.045, + "47.243046" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102408.05, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102408.055, + "0" + ] + } + ] + }, + { + "utc": 1795102413.071986, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102413.077, + "27" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102413.082, + "52.368544" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102413.085, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102413.09, + "0" + ] + } + ] + }, + { + "utc": 1795102418.1017191, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102418.105, + "27" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102418.11, + "52.368544" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102418.115, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102418.118, + "0" + ] + } + ] + }, + { + "utc": 1795102423.126356, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102423.129, + "29" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102423.131, + "57.523482" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102423.134, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102423.138, + "0" + ] + } + ] + }, + { + "utc": 1795102428.1532001, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102428.158, + "32" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102428.166, + "62.648557" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102428.171, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102428.175, + "0" + ] + } + ] + }, + { + "utc": 1795102433.191692, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "pending", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102433.195, + "34" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102433.201, + "67.747257" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102433.206, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102433.211, + "0" + ] + } + ] + }, + { + "utc": 1795102438.222282, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "pending", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102438.223, + "37" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102438.226, + "72.856874" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102438.229, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102438.232, + "0" + ] + } + ] + }, + { + "utc": 1795102443.238092, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "pending", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102443.239, + "39" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102443.242, + "77.986024" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102443.244, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102443.247, + "0" + ] + } + ] + }, + { + "utc": 1795102448.262121, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102448.266, + "42" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102448.274, + "83.109209" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102448.279, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102448.282, + "0" + ] + } + ] + }, + { + "utc": 1795102453.297589, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "pending", + "activeAt": "2026-11-19T15:34:09.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102453.302, + "45" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102453.307, + "88.220407" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102453.311, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102453.316, + "0" + ] + } + ] + }, + { + "utc": 1795102458.331528, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "pending", + "activeAt": "2026-11-19T15:34:09.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102458.335, + "47" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102458.34, + "93.351768" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102458.344, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102458.347, + "0" + ] + } + ] + }, + { + "utc": 1795102463.361185, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "pending", + "activeAt": "2026-11-19T15:34:09.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102463.366, + "50" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102463.373, + "98.473515" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102463.377, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102463.381, + "0" + ] + } + ] + }, + { + "utc": 1795102468.3923469, + "phase": "worker_stopped", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "pending", + "activeAt": "2026-11-19T15:34:09.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102468.397, + "52" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102468.402, + "103.563256" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102468.405, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102468.408, + "0" + ] + } + ] + }, + { + "utc": 1795102474.3263402, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "pending", + "activeAt": "2026-11-19T15:34:09.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102474.327, + "55" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102474.331, + "108.703388" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102474.334, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102474.337, + "0" + ] + } + ] + }, + { + "utc": 1795102479.345686, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "pending", + "activeAt": "2026-11-19T15:34:09.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102479.346, + "57" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102479.349, + "113.829774" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102479.351, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102479.352, + "0" + ] + } + ] + }, + { + "utc": 1795102484.361992, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102484.363, + "60" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102484.365, + "118.941931" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102484.367, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102484.37, + "0" + ] + } + ] + }, + { + "utc": 1795102489.383159, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102489.387, + "51" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102489.392, + "100.000807" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102489.395, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102489.398, + "0" + ] + } + ] + }, + { + "utc": 1795102494.410699, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102494.412, + "25" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102494.415, + "49.058342" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102494.417, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102494.421, + "0" + ] + } + ] + }, + { + "utc": 1795102499.42594, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + }, + { + "name": "AttachmentQueueLag", + "state": "firing", + "activeAt": "2026-11-19T15:33:04.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102499.428, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102499.43, + "0.110804" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102499.432, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102499.434, + "0" + ] + } + ] + }, + { + "utc": 1795102504.445472, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102504.446, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102504.449, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102504.451, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102504.453, + "0" + ] + } + ] + }, + { + "utc": 1795102509.467972, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102509.471, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102509.474, + "0.3242" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102509.477, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102509.479, + "0" + ] + } + ] + }, + { + "utc": 1795102514.484546, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadyFastBurn", + "state": "firing", + "activeAt": "2026-11-19T15:33:49.758725496Z" + }, + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102514.485, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102514.487, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102514.489, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102514.491, + "0" + ] + } + ] + }, + { + "utc": 1795102519.500073, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102519.5, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102519.502, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102519.504, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102519.506, + "0" + ] + } + ] + }, + { + "utc": 1795102524.521808, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102524.525, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102524.53, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102524.536, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102524.541, + "0" + ] + } + ] + }, + { + "utc": 1795102529.548005, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102529.548, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102529.55, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102529.551, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102529.553, + "0" + ] + } + ] + }, + { + "utc": 1795102534.569834, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102534.573, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102534.578, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102534.583, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102534.59, + "0" + ] + } + ] + }, + { + "utc": 1795102539.60006, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102539.601, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102539.604, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102539.605, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102539.607, + "0" + ] + } + ] + }, + { + "utc": 1795102544.619918, + "phase": "after_worker_recovery", + "alerts": [ + { + "name": "AttachmentReadySlowBurn", + "state": "firing", + "activeAt": "2026-11-19T15:34:09.758725496Z" + } + ], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102544.62, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102544.624, + "0.100515" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102544.625, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102544.626, + "0" + ] + } + ] + }, + { + "utc": 1795102549.641671, + "phase": "after_worker_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102549.642, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102549.644, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102549.645, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102549.647, + "0" + ] + } + ] + }, + { + "utc": 1795102554.66223, + "phase": "after_worker_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102554.665, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102554.67, + "0.396407" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102554.675, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102554.679, + "0" + ] + } + ] + }, + { + "utc": 1795102559.688737, + "phase": "after_worker_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102559.69, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102559.693, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102559.695, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102559.697, + "0" + ] + } + ] + }, + { + "utc": 1795102564.7094662, + "phase": "after_worker_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102564.71, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102564.712, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102564.714, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102564.716, + "0" + ] + } + ] + }, + { + "utc": 1795102569.7255301, + "phase": "after_worker_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102569.727, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102569.729, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102569.732, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102569.734, + "0" + ] + } + ] + }, + { + "utc": 1795102574.744999, + "phase": "after_worker_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102574.746, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102574.749, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102574.751, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102574.754, + "0" + ] + } + ] + }, + { + "utc": 1795102579.769393, + "phase": "after_worker_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102579.77, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102579.772, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102579.774, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102579.775, + "0" + ] + } + ] + }, + { + "utc": 1795102584.7893918, + "phase": "after_worker_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102584.79, + "0" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102584.792, + "0" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102584.793, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102584.795, + "0" + ] + } + ] + }, + { + "utc": 1795102589.800083, + "phase": "after_worker_recovery", + "alerts": [], + "queue_depth": [ + { + "metric": {}, + "value": [ + 1795102589.801, + "1" + ] + } + ], + "queue_age": [ + { + "metric": {}, + "value": [ + 1795102589.804, + "0.315399" + ] + } + ], + "api_bad_30s": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:30s" + }, + "value": [ + 1795102589.806, + "0" + ] + } + ], + "api_bad_2m": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "value": [ + 1795102589.809, + "0" + ] + } + ] + } + ], + "requests": [ + { + "utc": 1795101975.3640609, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.010765250015538186, + "trace_id": "20da6363ba4016884db3676a66020f91", + "request_id": "f2867d8a-9f34-43a5-bfe2-eec1be1f94f7" + }, + { + "utc": 1795101975.3643572, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.019703167024999857, + "trace_id": "ba2ce980d70cc985b88e67af3db1ced2", + "request_id": "9b0571bd-f17f-4e68-beb1-92bb8bfa1612" + }, + { + "utc": 1795101975.869212, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.007860374986194074, + "trace_id": "4e8e53ae4fbe7868fecce5e8176ce727", + "request_id": "669b2122-2cf8-40d0-9682-d089f54a3904" + }, + { + "utc": 1795101976.3724842, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.02430474996799603, + "trace_id": "e684f1aef11f463318659fd9e38eac2e", + "request_id": "4ae7518c-fdfa-4456-9357-cb6de213cf04" + }, + { + "utc": 1795101976.86507, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.024168082978576422, + "trace_id": "bbb8d6d8354dd6954c75a071ceeaf5c7", + "request_id": "3576e0e4-4280-4b9f-8d50-aafb19524a76" + }, + { + "utc": 1795101977.3741581, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.02336287498474121, + "trace_id": "dd1d89e2b18fa296ed33b1d866c817af", + "request_id": "89d4e7af-be8f-4f59-a87e-c285fe6cca05" + }, + { + "utc": 1795101977.3741891, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03709350002463907, + "trace_id": "b37372bfa714549fac70b101d8d108ee", + "request_id": "437d4580-b063-4168-9bfe-93719db747e2" + }, + { + "utc": 1795101977.872333, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005583999969530851, + "trace_id": "485d2121f06953f25ce777654c3ba7fa", + "request_id": "3a71712f-dad3-45d1-8a33-869f43a23b69" + }, + { + "utc": 1795101978.374162, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.02474829100538045, + "trace_id": "e01b47634fb5920282ceae3449ee593a", + "request_id": "96590cc6-8203-4dc0-a361-44fbb85578fa" + }, + { + "utc": 1795101978.865813, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01839379098964855, + "trace_id": "887e9ce4a6c455cd63a4e0c0a12efa64", + "request_id": "1aa2fe18-a747-4a83-bb41-73f6fcdc5444" + }, + { + "utc": 1795101979.365311, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01838187500834465, + "trace_id": "68b3e0c2f9888c4dee375e9d671237b4", + "request_id": "747e435b-5972-4d56-a74c-5a9b03095bd2" + }, + { + "utc": 1795101979.365351, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.032965334015898407, + "trace_id": "656d0a3c8c2a21582727ed15da86ee3c", + "request_id": "18095228-a62a-4598-9785-f707b7bf6335" + }, + { + "utc": 1795101979.868278, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005580083001405001, + "trace_id": "231ba6b2edaae9c97db8a42c75609bba", + "request_id": "d151e574-90fa-466b-b8db-ba42839523c8" + }, + { + "utc": 1795101980.3718731, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016335749998688698, + "trace_id": "4280d71ce1c7acc48e69ca05fd029a04", + "request_id": "92b9d58e-6a18-4437-92a6-04be3b048f02" + }, + { + "utc": 1795101980.870923, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.013982250005938113, + "trace_id": "3350064c56f8b9167b899a7c31ed320f", + "request_id": "496d7a74-c289-46c1-b46f-accb94ce1f70" + }, + { + "utc": 1795101981.373081, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.019667666987515986, + "trace_id": "5e7b230b6f74b1b1223bc08b09ef07e7", + "request_id": "ff84e0ee-1fce-41ab-ad42-d1cda7b97c27" + }, + { + "utc": 1795101981.37312, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03135025000665337, + "trace_id": "1d73fe090a2cc7e9d88fe02d3d3cf7a1", + "request_id": "09ae496d-d5de-41ba-adfb-9ec314961cb1" + }, + { + "utc": 1795101981.872093, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.0068066660314798355, + "trace_id": "78294c690dfdf630129a40d2266280c3", + "request_id": "97f00fd8-a365-4f73-b845-826670414a92" + }, + { + "utc": 1795101982.3723369, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016980250016786158, + "trace_id": "1243b5529a905ad3a2d27e6bf7106115", + "request_id": "ca85623c-53f4-439d-b6dc-fe5fc82f1e9d" + }, + { + "utc": 1795101982.8720498, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01968954096082598, + "trace_id": "2579a864f32f640505d91d38c21a89be", + "request_id": "de501065-9a98-44e5-a987-3621212c93ec" + }, + { + "utc": 1795101983.370246, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.019514874962624162, + "trace_id": "cc62a83642e4e08e078ad5664c8d395e", + "request_id": "9aa17f78-f70e-434d-8485-3325d59ec243" + }, + { + "utc": 1795101983.370286, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029693457996472716, + "trace_id": "805231a5deff9adf10db5e071f5b16a7", + "request_id": "9d079d7b-4b23-4572-91dc-faf63839842f" + }, + { + "utc": 1795101983.864497, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005852582980878651, + "trace_id": "c89238514a1914c9733c0479476c2365", + "request_id": "92496018-5b32-48d9-9143-dcb8a3cb13fd" + }, + { + "utc": 1795101984.372339, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01012562500545755, + "trace_id": "391868db55518cca3df43e6bbc4223be", + "request_id": "743b3edc-cd9b-449d-beec-7674c79ba9b9" + }, + { + "utc": 1795101984.870369, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018680665991269052, + "trace_id": "9e75c5f2c8f694f7ffa75533742f89d6", + "request_id": "67128c72-b0ee-4263-8ed6-04d6d4df2a1e" + }, + { + "utc": 1795101985.369725, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01931104197865352, + "trace_id": "231b2e1dc07e9ec63c0c1cf92bceb33c", + "request_id": "7ab078eb-3d0b-40ff-9bc1-c14962bf611e" + }, + { + "utc": 1795101985.369764, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030933083035051823, + "trace_id": "e6e4d035f678046af442218bbceb865b", + "request_id": "b0baac40-357f-4f97-91bd-1008669a7b77" + }, + { + "utc": 1795101985.865477, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.0055495830019935966, + "trace_id": "b265003a3d32889c3f423855ec3b10ff", + "request_id": "71e42f49-b351-452f-8663-eb81ccb90098" + }, + { + "utc": 1795101986.372278, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006783749966416508, + "trace_id": "3a366ad69c778b54ab2ff0a881f4acf2", + "request_id": "7e83f0e6-68bb-4b38-8269-fc0d8536ee57" + }, + { + "utc": 1795101986.8703501, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017052542010787874, + "trace_id": "19a43a2aa785a7d4d369b04b718b03cb", + "request_id": "b2e13609-49ce-461d-98a7-8cedeed1dd23" + }, + { + "utc": 1795101987.365941, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.020128959033172578, + "trace_id": "f30e11024ec81e485d398fb93ab43b7a", + "request_id": "6534ef98-a93c-41f9-a01d-a6979d5e59f9" + }, + { + "utc": 1795101987.365964, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.034320332983043045, + "trace_id": "d0bada8814ed340aac5c18b9f2ccb3dc", + "request_id": "b660e08b-e74a-4a77-98ea-fb2cf6c62d68" + }, + { + "utc": 1795101987.869478, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005460791988298297, + "trace_id": "25113189edebacd2b144861e18302b54", + "request_id": "a44d2784-87d5-47dc-a2c0-71a15d0cd808" + }, + { + "utc": 1795101988.3670342, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016954833990894258, + "trace_id": "6586c06e669a7a806fb807ac98258fb1", + "request_id": "b715c006-bf08-45d5-a39c-adac52992012" + }, + { + "utc": 1795101988.8741398, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017008833994623274, + "trace_id": "0c47dbed3219f17dfc9232a12c94ec6b", + "request_id": "e625a81d-a6f1-4c93-b10a-5ca8ca28c691" + }, + { + "utc": 1795101989.367188, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018850207969080657, + "trace_id": "861143d8404bbb207863d9e9f8f76a78", + "request_id": "72352801-490e-4fb9-9671-1abf3cde87a8" + }, + { + "utc": 1795101989.367229, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.0298374590347521, + "trace_id": "4d365cd490655d37c9f3541d0afad1fd", + "request_id": "786b8492-b584-40d6-b76f-f994d0e4cfd7" + }, + { + "utc": 1795101989.8722591, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005293000023812056, + "trace_id": "14cf2927dc8f935bc0514872aebe013d", + "request_id": "8701fe31-b1a5-43d4-a7c4-9cf4e96e0adf" + }, + { + "utc": 1795101990.3723059, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01029291603481397, + "trace_id": "3319cf2c4bfb92307c458fc5f2b87d54", + "request_id": "aaa30738-5fe3-4142-ae8d-824c262e12f3" + }, + { + "utc": 1795101990.8689191, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.015075540984980762, + "trace_id": "c0572bec48d499fe60336f4ab457fa3b", + "request_id": "63925144-0e87-492d-87fb-09dd81522ee1" + }, + { + "utc": 1795101991.3703291, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016907375014852732, + "trace_id": "863dd89fd200ceaa134cb80582d386bd", + "request_id": "3243978c-1b59-4b2f-8628-cd19e7d13d69" + }, + { + "utc": 1795101991.370367, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028556875011418015, + "trace_id": "9fda2fefe104fb2f3d38c97c3e151b7d", + "request_id": "02fc8a12-eeb2-40c2-9321-00b73abb19f2" + }, + { + "utc": 1795101991.8717499, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.00693120900541544, + "trace_id": "68fad7ecc96745b5c781059cf106b171", + "request_id": "2b2da88d-6037-42ea-9558-210f2930ec5d" + }, + { + "utc": 1795101992.372948, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.0161244579940103, + "trace_id": "852028c8e19c09e90aad4bca714cb6a3", + "request_id": "39421210-6aef-4407-965b-8734086d5d9b" + }, + { + "utc": 1795101992.874131, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.007538792036939412, + "trace_id": "5fa25799d0a264ce084a7391cf1305d1", + "request_id": "fa8c5658-b501-4f44-8d24-ef4b7d525aa5" + }, + { + "utc": 1795101993.366493, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.011998708010651171, + "trace_id": "ada0a3cf6b79625e3b84ab80b7458757", + "request_id": "275b71ef-3def-48dc-8e0a-1fa9bf02e0dd" + }, + { + "utc": 1795101993.366532, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.021283833018969744, + "trace_id": "2f692ca07ebfd4138e191da38d73bd56", + "request_id": "6fa64557-4dea-4158-80e1-9f56b29b78be" + }, + { + "utc": 1795101993.8719969, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.008297541993670166, + "trace_id": "eaae237708b542fa6672172c7f8bfaab", + "request_id": "bb5fb217-c013-4952-a1ef-59248602e76c" + }, + { + "utc": 1795101994.370305, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016472916991915554, + "trace_id": "0eb62642b194837a73cf9074daa34d18", + "request_id": "96f807c8-1477-4e53-8443-09173434f421" + }, + { + "utc": 1795101994.871888, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.0061087909853085876, + "trace_id": "0d524868b83d9b85646b89248165d109", + "request_id": "9ed6acbc-7dba-4750-b15c-9a601dfa2174" + }, + { + "utc": 1795101995.370327, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.00949516601394862, + "trace_id": "113e56d2bff1089b398079d32c258c10", + "request_id": "e1473d1b-6cb9-4612-b6a5-7b3b9fcb4789" + }, + { + "utc": 1795101995.3703659, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.015667707950342447, + "trace_id": "e9fa5b7ffacb980f2c9702d41444ed0a", + "request_id": "821f4361-15f0-4ef7-9732-24c3371b048d" + }, + { + "utc": 1795101995.8741, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005212999996729195, + "trace_id": "179d91d7f7675e434c4579904f857571", + "request_id": "8aeb3df0-1a2b-4a09-902c-89102ff7c4ee" + }, + { + "utc": 1795101996.372965, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018748374946881086, + "trace_id": "4429c99bfe7b85102d32253191433134", + "request_id": "eb0c77b4-4f50-454b-b56d-9e12ff14fa94" + }, + { + "utc": 1795101996.8699899, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.015839791973121464, + "trace_id": "4325572a6086ccacc6c5fbd3b66dfc21", + "request_id": "68a52828-8e5f-42d2-bb67-13527696422d" + }, + { + "utc": 1795101997.374084, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.007237874960992485, + "trace_id": "9af5788dceffa5371c35d74ebae4ee2b", + "request_id": "bef2d22d-314b-4bd6-8d3f-217a42d1de93" + }, + { + "utc": 1795101997.3741162, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01253366598393768, + "trace_id": "b170b0c763256146c7e2454f6a8bd935", + "request_id": "2b7cd320-680f-4835-89b7-1c0930b01abe" + }, + { + "utc": 1795101997.870234, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005434208957012743, + "trace_id": "b49827e58856c7d8ac911db3fcf9b7d0", + "request_id": "b1d50bad-07bd-43e6-ab65-4e565cd1a097" + }, + { + "utc": 1795101998.365003, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01653583301231265, + "trace_id": "ee166517a32825aa61b291431b4a5762", + "request_id": "6bd455dc-25e2-4f1b-bd0b-7fef844e91d7" + }, + { + "utc": 1795101998.874104, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01739462499972433, + "trace_id": "ca3879ffacfa3608e0357e7e9fbe9194", + "request_id": "b2673f7a-c0ac-4f61-acdc-c9e6d6373a94" + }, + { + "utc": 1795101999.372298, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.0077991249854676425, + "trace_id": "cf8b5720378f4bf2e453007ba40d52ce", + "request_id": "4f27b0ab-be75-46d0-8b4c-f44509e32c06" + }, + { + "utc": 1795101999.3723361, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.014051250007469207, + "trace_id": "feea1aaca56790c5d24df2defcf6f3c1", + "request_id": "622c8f6c-a23b-4f11-a679-d1efe8b7d025" + }, + { + "utc": 1795101999.874052, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.011651874985545874, + "trace_id": "e5c56e609c3f1c096bbb8e0b2921be87", + "request_id": "76ac19f1-3472-4626-b490-05426da538ef" + }, + { + "utc": 1795102000.3708138, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.015982791024725884, + "trace_id": "6af3501e768c735c522fce4bd867c72a", + "request_id": "aae6b321-e2e1-4cdf-bfbb-31f2511ddb66" + }, + { + "utc": 1795102000.874094, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.009920333046466112, + "trace_id": "3237a8c755edf972ca8f009b6b2730d3", + "request_id": "98f34832-35f7-4b56-b5d0-02ab18fc3b72" + }, + { + "utc": 1795102001.3657038, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01849516702350229, + "trace_id": "c2690e3cb392327c564c0ab458b304fb", + "request_id": "92608a4d-0e87-4631-a6df-7612361b2c65" + }, + { + "utc": 1795102001.365716, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02528579201316461, + "trace_id": "dfc9adc71e95d77513887dfbbb95fe12", + "request_id": "cec755c1-a94f-429a-9f38-34a731e2f035" + }, + { + "utc": 1795102001.87224, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.008035583014134318, + "trace_id": "8ac276c7b255427120d0d316a19e1d8e", + "request_id": "ceff1532-da7e-4c16-9f82-7a7267aeba83" + }, + { + "utc": 1795102002.370282, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.02005366695811972, + "trace_id": "5ca998e3312d81f28301bbccf83f3219", + "request_id": "585c60c9-928c-44e4-89c7-207d0f372676" + }, + { + "utc": 1795102002.8677828, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01858787500532344, + "trace_id": "f6c3a187dfd10321b894b322472cfe03", + "request_id": "e529ca75-7b39-4962-9cc0-1a6b37476104" + }, + { + "utc": 1795102003.3722932, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.035887249978259206, + "trace_id": "992662c8aa33a9458cbc99a56694d307", + "request_id": "932438c4-43e5-47c8-9902-b05e334c7a35" + }, + { + "utc": 1795102003.3723369, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.04704891599249095, + "trace_id": "0a7cab58ea742a3832dcaea5148bfd5e", + "request_id": "4fa940bc-c1d1-4485-bd3d-dc1a5b51c16a" + }, + { + "utc": 1795102003.870262, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005699250032193959, + "trace_id": "99cc744118a6c7ca948eff67ba2e4aba", + "request_id": "c1a5d01f-389d-4216-bbfd-809c1656bd31" + }, + { + "utc": 1795102004.372266, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016071708989329636, + "trace_id": "f9d57f3148983166450866ec3417030b", + "request_id": "1552d640-ac11-48f0-bc0e-95cebdf69d8a" + }, + { + "utc": 1795102004.870281, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018192040966823697, + "trace_id": "b951ef733dd6d2ad4d19d50597d62352", + "request_id": "a787f432-24e3-4579-a99b-88c0fd2f9b7d" + }, + { + "utc": 1795102005.3722382, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.022749583004042506, + "trace_id": "985e9995327b7340758fea9af13d5331", + "request_id": "a581d83e-48c1-4f13-bab0-7c7086d0b243" + }, + { + "utc": 1795102005.3722649, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030377082992345095, + "trace_id": "3e344d90a940be368716f1a9cdec5eee", + "request_id": "1aca271e-33b7-426f-8af6-016059e46fa5" + }, + { + "utc": 1795102005.874089, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005925749952439219, + "trace_id": "b44f784ba4e00045f4b118fcd1896a6e", + "request_id": "5fa8595f-f788-4f4d-a3f2-bb656485a18a" + }, + { + "utc": 1795102006.3702788, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018084791023284197, + "trace_id": "727eaaf1b3742d65c1ebd03a4a2dea5b", + "request_id": "acc016ce-8ad3-4ad3-b32f-6523cd5f6a6c" + }, + { + "utc": 1795102006.872854, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.019095999945420772, + "trace_id": "568aca0fe790fc98cb770e614accd74e", + "request_id": "8af1adc6-496a-4f2f-b363-8e10b9e7bc72" + }, + { + "utc": 1795102007.370293, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01848341600270942, + "trace_id": "59f71a1068c3b33d35d875f0ce5a5170", + "request_id": "ab42ddf4-25dd-45d8-b73b-ffd0a2346774" + }, + { + "utc": 1795102007.370328, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03126616700319573, + "trace_id": "7aca8a9e7cf7f5545ab7dffd5d89e873", + "request_id": "79923cad-2fd9-4b16-8c41-c7b5a1fd66da" + }, + { + "utc": 1795102007.866282, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006331834010779858, + "trace_id": "e0412d1f79b8e361134dfed953f040b6", + "request_id": "05528291-ded4-41fe-8d2b-f02b9ca86143" + }, + { + "utc": 1795102008.370255, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01645845896564424, + "trace_id": "290f774e9488887f9039eb479971bf77", + "request_id": "da6ea3e3-c67a-493f-8ceb-3f5915aaa192" + }, + { + "utc": 1795102008.867615, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017143125005532056, + "trace_id": "edc0240ebe20e4397b5b2df1abfaa7d3", + "request_id": "ad680c53-f27c-43de-8133-70309b5172d6" + }, + { + "utc": 1795102009.3722649, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01800520799588412, + "trace_id": "afc86caaf01650fd1357ee794452cb30", + "request_id": "bee748fa-9d33-4b84-a400-7d045b8e7257" + }, + { + "utc": 1795102009.372309, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.031246625003404915, + "trace_id": "90382398d5c21dc47926a30ce14cb36f", + "request_id": "1f2a2b47-2471-4f44-9e34-748e801c19d5" + }, + { + "utc": 1795102009.8721652, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005314250010997057, + "trace_id": "1cf0d8f95ce5f231882426c498572c48", + "request_id": "22a5c186-4979-42a3-852a-f9f7353f3ea0" + }, + { + "utc": 1795102010.366414, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01650720799807459, + "trace_id": "4daee4f04cf73311295852de48c9a444", + "request_id": "48dba9a9-36a1-4f32-9e6f-b4be5955e65e" + }, + { + "utc": 1795102010.8722801, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018341541988775134, + "trace_id": "b2366fedda877833e5ca60e7012600ec", + "request_id": "33508dc5-624f-4a96-86dd-c2c2ebfd9e53" + }, + { + "utc": 1795102011.370733, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01967408397467807, + "trace_id": "cd057c41b79f00af8de97fe4f8f1e9e2", + "request_id": "19d835f7-7b6b-406d-8a40-6f6b624cf3f1" + }, + { + "utc": 1795102011.370775, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030376249982509762, + "trace_id": "77e6e8a03667b50083c3ab1052129e45", + "request_id": "7f89b7e8-9ee2-42fa-a416-94cacdb0a958" + }, + { + "utc": 1795102011.87413, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006562083028256893, + "trace_id": "9351dfca5cb673a585eeebded527085c", + "request_id": "b42364ae-cdad-4eff-baca-c24b31756237" + }, + { + "utc": 1795102012.37406, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.010946291033178568, + "trace_id": "ef482cdc61debcc092e37539be612c46", + "request_id": "ef9d5543-aa26-4d7d-a26e-7604624f468f" + }, + { + "utc": 1795102012.872232, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01644908299203962, + "trace_id": "fd29b0cf39fb43ee984b8b3119d73d3d", + "request_id": "a9683ca9-51a2-44d9-8181-f22692cd6ec9" + }, + { + "utc": 1795102013.372311, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01974491705186665, + "trace_id": "d74d4c3da1cee4b1d529c16a8e2716ed", + "request_id": "3ec3a011-06ce-4c1b-b5cf-d1d72dc5f5c9" + }, + { + "utc": 1795102013.372362, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029093624965753406, + "trace_id": "7717dff50f51c14cc1b7aad0679444df", + "request_id": "7b3588bf-9033-439d-a93a-a0459838cbab" + }, + { + "utc": 1795102013.874075, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.0054229170200414956, + "trace_id": "d8a21abc4aa5a8174cfe6321e0ec263c", + "request_id": "8b31cad4-38e1-4e7a-813a-756822d9900b" + }, + { + "utc": 1795102014.372142, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.0060848339926451445, + "trace_id": "497ea0a45467fc3c301fda6ed0668111", + "request_id": "f18d81f0-a410-4426-80f6-6f97652a7d37" + }, + { + "utc": 1795102014.870215, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.015361166966613382, + "trace_id": "d130d1c8e40850712af36172ffdbadea", + "request_id": "2406c13f-a82f-4ca4-b1d9-0449fe8f9410" + }, + { + "utc": 1795102015.371993, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.019305249967146665, + "trace_id": "6b07ec1b594e69d1d35e11401a47d3c5", + "request_id": "56230527-1adf-4dde-a63d-fd9e68380e37" + }, + { + "utc": 1795102015.372027, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030091541993897408, + "trace_id": "c97031d87362b9ffd9ce0a1134ee3b58", + "request_id": "053091bd-6f1a-4846-8390-b015a8ee85c7" + }, + { + "utc": 1795102015.868064, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006930542003829032, + "trace_id": "4b8537e3834b8af235389886c1416e90", + "request_id": "abfc247c-35df-45ec-b796-f4498696c085" + }, + { + "utc": 1795102016.370236, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.020750041992869228, + "trace_id": "5c7e615b1fb99609f6437eecf4a260f4", + "request_id": "3d21e639-e123-4296-a913-30ca349c8c6e" + }, + { + "utc": 1795102016.872163, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.014324375020805746, + "trace_id": "5b501867b3e302a3eafea0bbc623a2ae", + "request_id": "ecb329a7-f6db-44cc-8db9-a5f70811e79d" + }, + { + "utc": 1795102017.373995, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017552292032632977, + "trace_id": "b9c6578290459f56d3eabd913854c837", + "request_id": "af87d144-5512-49a2-8cbe-77124ad4ac16" + }, + { + "utc": 1795102017.374031, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030125166988000274, + "trace_id": "aee3437b1f576cbd0808490596ed6c88", + "request_id": "e72e732a-87fe-4ddd-a73f-da20326dd0b0" + }, + { + "utc": 1795102017.870156, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005638125003315508, + "trace_id": "5b3a7a7f46479b372d877c41d23e00e7", + "request_id": "c13fa07d-8377-4f48-bc8e-ef1f617d4ef3" + }, + { + "utc": 1795102018.365668, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01841141702607274, + "trace_id": "b75998cddae72e626463937f06fad8e9", + "request_id": "44193fb9-3e02-4346-bb47-774d229b0f22" + }, + { + "utc": 1795102018.872204, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.00950154202291742, + "trace_id": "1c5d085ba6c7cd5a86dcde7d22fbdc30", + "request_id": "68671640-b7db-479d-87f4-27c3efb65ed6" + }, + { + "utc": 1795102019.374036, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.019035209028515965, + "trace_id": "be130bb1da33ac562815e3c8f719c1e1", + "request_id": "55f0c1bd-3c3f-438a-b31e-a068da1164ff" + }, + { + "utc": 1795102019.3740768, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030059457989409566, + "trace_id": "41a9ecb456a862abcb4cd1183e3f7fe7", + "request_id": "4b70b16c-1f42-4e68-b14f-5b25cd774cb1" + }, + { + "utc": 1795102019.8721201, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005605833022855222, + "trace_id": "9d0759557b503688023e12cf93fc42dd", + "request_id": "fbac1012-3af1-4356-9f3f-6741bdf4d126" + }, + { + "utc": 1795102020.372329, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.0106348330155015, + "trace_id": "3c211f57a493de099f0071f5a0481774", + "request_id": "59fc01e3-3c2f-40ed-a7d3-593a83824eba" + }, + { + "utc": 1795102020.865249, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005588917003478855, + "trace_id": "53be6ea553426f190b858523facd5464", + "request_id": "0b1fd23e-84ce-42d8-9c53-a6985c61f08c" + }, + { + "utc": 1795102021.3722088, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.02124299999559298, + "trace_id": "3ba9b8a77f85d1f7d9e655e1323f2505", + "request_id": "70a4942a-2055-41cd-a95b-7c314321c741" + }, + { + "utc": 1795102021.372246, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03055187501013279, + "trace_id": "ec330fcdc314e0afb3d74bdfa8475d58", + "request_id": "45ab13e7-a823-4473-b6f8-364bc1921535" + }, + { + "utc": 1795102021.872235, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.007953458989504725, + "trace_id": "8c9d38afe3605c5fed655728de7935bf", + "request_id": "76b9d809-8380-49b3-b85d-0bd4026afb79" + }, + { + "utc": 1795102022.370655, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017351375019643456, + "trace_id": "b6ded909e1c25fa77f14f4c0f9984b65", + "request_id": "88bdcb0f-8648-48df-9277-a3f84f7e605e" + }, + { + "utc": 1795102022.8740082, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.019088083994574845, + "trace_id": "a672cbf5a84bd4dd15946acf31268efe", + "request_id": "d3aee7d4-208e-45b8-a21f-b3770ccd33bc" + }, + { + "utc": 1795102023.37221, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018383125017862767, + "trace_id": "071f70497580a6f9fd9c97bcb7d33b90", + "request_id": "72337d4c-4b71-40c4-a60d-9692919a2185" + }, + { + "utc": 1795102023.372255, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029196083021815866, + "trace_id": "a37e97293c2ce13155f9cdc5d2f62133", + "request_id": "bfae468f-4293-415c-9153-27d2938a9f96" + }, + { + "utc": 1795102023.873858, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006350999989081174, + "trace_id": "80c0b1bc3a3360087db184fbe727108f", + "request_id": "acebdc8f-3db7-47e3-9fbb-534bd3d72d8f" + }, + { + "utc": 1795102024.373987, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01783550000982359, + "trace_id": "1041ef5f374cc69f05afd2c5e783b23b", + "request_id": "e05d56da-775c-4597-9034-dbcf69b48262" + }, + { + "utc": 1795102024.870212, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016198083001654595, + "trace_id": "6adaf76bb4a328acc6504277d5373193", + "request_id": "53fcf211-e426-4ff9-861a-32f23fed998a" + }, + { + "utc": 1795102025.366365, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.011140000016894192, + "trace_id": "b490f1e2998cc5f1b6c6de5303e709a3", + "request_id": "aba11f84-3067-4976-bcd1-698bfd18c5bd" + }, + { + "utc": 1795102025.366403, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.019037874997593462, + "trace_id": "879d0641e15addb93ce63228b46e7700", + "request_id": "118d30de-2d78-47cd-9928-7a0487ba2dd3" + }, + { + "utc": 1795102025.870124, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.0056422079796902835, + "trace_id": "634a5643aafbe35d2c240e3463a66b49", + "request_id": "a119c0e7-3ffd-4246-9d18-5efbacc05da9" + }, + { + "utc": 1795102026.368215, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01711383304791525, + "trace_id": "6e6d34def15b67314dd855bde972be1c", + "request_id": "f18cd471-8cab-4aa5-a58d-03fc018ceb1c" + }, + { + "utc": 1795102026.8702168, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016709582996554673, + "trace_id": "1c6106e08bd5f1c160a5a28186e2f5fe", + "request_id": "0cb27e54-c6ae-4c85-ad8a-3228c19ece0c" + }, + { + "utc": 1795102027.372043, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006802125019021332, + "trace_id": "5ea9e78c275e9ee4c0d5f3322111327c", + "request_id": "11cccff8-3224-468f-aeec-34e0234a97cf" + }, + { + "utc": 1795102027.3720598, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011626082996372133, + "trace_id": "338c967307c2c8f005ae3bb8d0135658", + "request_id": "4fee9b2c-adbe-455a-bebf-833ae8080997" + }, + { + "utc": 1795102027.864618, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005919375049415976, + "trace_id": "d7924d30bdba2ee366af90e07788f1e1", + "request_id": "caa2f061-88ca-482f-bb07-eb39641b9598" + }, + { + "utc": 1795102028.37216, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016105291026178747, + "trace_id": "89bd6d0c66923e4ab1710d9ab84091e8", + "request_id": "676c1833-bd3c-4970-a08c-a410311ec5dd" + }, + { + "utc": 1795102028.872192, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017783874995075166, + "trace_id": "446882b1228628d88e9344fad2d55837", + "request_id": "26e3266d-361d-4dac-9915-a442da2fbcc4" + }, + { + "utc": 1795102029.367399, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.021357082994654775, + "trace_id": "9f82ec109dbe4177435cc5a1222ed905", + "request_id": "67283a26-d7ad-47d1-aaa4-883ebb885ba1" + }, + { + "utc": 1795102029.36744, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03448083298280835, + "trace_id": "ff7b1d02252edfb2c3f0b9c002334674", + "request_id": "f7e46c54-3426-49a4-87dd-e532aed73575" + }, + { + "utc": 1795102029.867607, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006501291994936764, + "trace_id": "9385e0bd4732897daa4d1532d8413a1e", + "request_id": "79bd6a02-83b2-446c-a784-e36eb4ef025f" + }, + { + "utc": 1795102030.3640478, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01593466696795076, + "trace_id": "de1178f017d17dc4a2074abec36b120d", + "request_id": "121ad6bc-ed35-4806-8cf7-0589efb7bdae" + }, + { + "utc": 1795102030.873914, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.007101707975380123, + "trace_id": "d6cab82c88a81613095be742ced045a0", + "request_id": "624beb11-e2fe-4976-ab0e-70c53c29d405" + }, + { + "utc": 1795102031.37024, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01817054202547297, + "trace_id": "58d85025d4f49d4a271f4516cbc40186", + "request_id": "89ea2c25-c86c-447b-a68a-ed9785967ca4" + }, + { + "utc": 1795102031.370281, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029617750027682632, + "trace_id": "8ee882299a1bc2c5d4e011c4d4c6c771", + "request_id": "a24a8297-aefd-4e69-a21a-0cf55544b25e" + }, + { + "utc": 1795102031.8739898, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005754999991040677, + "trace_id": "8becd7e064f801394939b0aefa64f3f1", + "request_id": "408f7344-6eae-4da5-86b1-9bcf598080c7" + }, + { + "utc": 1795102032.371441, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016658333013765514, + "trace_id": "6283512a70fa08479d8879d106b52a26", + "request_id": "ae0a9a03-fda4-418b-ba21-07b5ba3c4772" + }, + { + "utc": 1795102032.866317, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01303054200252518, + "trace_id": "babb9803c6809cad7ef0d6816d9fe0fd", + "request_id": "1aae5cda-5089-4727-97d7-9a544a84ac2c" + }, + { + "utc": 1795102033.3701901, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016606541990768164, + "trace_id": "d05068555b257f4c37c88d08976f95c5", + "request_id": "78027041-744a-4cc8-af7d-5c03cb61c9f7" + }, + { + "utc": 1795102033.37023, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.0294159579789266, + "trace_id": "c4304ebd4b204dcaf71e55c982e0e134", + "request_id": "f1529cff-ec46-4013-a2b1-f0048a788b2b" + }, + { + "utc": 1795102033.864521, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005554542003665119, + "trace_id": "d73e8cf01bd924c6e37b21e7c500da69", + "request_id": "0b9aa78a-5d40-426e-96aa-fec39ce4f39a" + }, + { + "utc": 1795102034.370454, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016150666982866824, + "trace_id": "237de8a2b020d7cb1c4815e44a590dd9", + "request_id": "a2e550d3-53c8-4cee-8303-aa93c6712e19" + }, + { + "utc": 1795102034.871346, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016289624967612326, + "trace_id": "d63469c5533c0a5a4ad5639a9451e969", + "request_id": "1de228a3-f820-4460-a2ad-814421356a43" + }, + { + "utc": 1795102035.367452, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.024341957992874086, + "trace_id": "06c25727e5455615bc6a67d95e8fe387", + "request_id": "15224b5e-dd08-40f2-be91-ba06c9add010" + }, + { + "utc": 1795102035.3674922, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03215004096273333, + "trace_id": "5cd9ec7e4e929504706e3e3c90f447c9", + "request_id": "1230745a-34a8-42d6-934c-dfac24c135b7" + }, + { + "utc": 1795102035.87264, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.013765999989118427, + "trace_id": "07034275ab3028ed87ded59d28a80589", + "request_id": "305789c4-23e9-4c81-a5e2-cdcffb6c384b" + }, + { + "utc": 1795102036.3721821, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.019726083031855524, + "trace_id": "a8e6afd4ad4da7b47b848a7218411f78", + "request_id": "fceb7efd-14f6-4367-8308-3cc202309e54" + }, + { + "utc": 1795102036.872133, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017617790959775448, + "trace_id": "a4344096cf05aa176028ebaf760c17ec", + "request_id": "185624f1-0f14-42b1-8daf-37d57226942b" + }, + { + "utc": 1795102037.3701801, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01897220901446417, + "trace_id": "b2ce84f7672bcd8989a68f8f6c6a9ff8", + "request_id": "97fe0de1-dc19-435b-a6aa-577ccd75af16" + }, + { + "utc": 1795102037.370219, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03189029195345938, + "trace_id": "526ec484c5c759fd2bc1e55f545789de", + "request_id": "a0e9b9f3-3b90-4f55-85d7-29fc67ff0b0a" + }, + { + "utc": 1795102037.872066, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005211625015363097, + "trace_id": "632348048705bfaa5dcc7fd41755e97e", + "request_id": "15984a43-1e53-4670-a1b2-651ef755b56b" + }, + { + "utc": 1795102038.3721259, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016495332994963974, + "trace_id": "a08d547f82376d4d94292ee10f5adc94", + "request_id": "729214bf-9210-4337-b03a-4da3b211ced8" + }, + { + "utc": 1795102038.8707242, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01620445802109316, + "trace_id": "44b278085cd6b295c2e92bfa2f372a5c", + "request_id": "5b8b8243-ba07-482d-9e6c-30eb4ab18bf5" + }, + { + "utc": 1795102039.3727648, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017808084026910365, + "trace_id": "1e201d795979ac2ba12edcd2e997e34e", + "request_id": "002bb3a7-b1ba-4883-8b0c-e6309e7ae524" + }, + { + "utc": 1795102039.372799, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03093433298636228, + "trace_id": "82870e032c495f78a42e713a38c031a6", + "request_id": "056d1f4c-0d88-4bd5-8e9d-59d829e42eec" + }, + { + "utc": 1795102039.864624, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005458583007566631, + "trace_id": "c37c60ca27937d7ba2eba4b6d29e2489", + "request_id": "56115ef0-2dba-44e7-911f-a0d8bb73b0b9" + }, + { + "utc": 1795102040.370115, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.008406415989156812, + "trace_id": "e26147b53f745eac16813ecb4edbe281", + "request_id": "61e11c18-1163-4c12-bdad-89f15fb4b574" + }, + { + "utc": 1795102040.8646579, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01634504203684628, + "trace_id": "bd0c334a83b8504732aecdaeee29bc3b", + "request_id": "67dfbbb4-47b6-4674-beb3-35930c7a0d07" + }, + { + "utc": 1795102041.36544, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01876754203112796, + "trace_id": "735e51e5e3fe135d3ac439c6c52a4b27", + "request_id": "a65d9ecc-9765-43c4-bd39-c2788ddaadb0" + }, + { + "utc": 1795102041.365462, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.031947958981618285, + "trace_id": "eaa77a1908241af70b21f64598bc59d2", + "request_id": "b9b492e0-6ff2-4e35-a7d4-bef0b96e965a" + }, + { + "utc": 1795102041.87397, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.009381374984513968, + "trace_id": "acea80f39d3c33076471ff09d7eb9c1d", + "request_id": "76d591fd-dd43-41c9-a693-409f29920e76" + }, + { + "utc": 1795102042.37007, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005422791989985853, + "trace_id": "b590f6392ae593f23131e2a38172cdc3", + "request_id": "1857d3ae-e8e0-484b-9143-a41970a26446" + }, + { + "utc": 1795102042.8706079, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017041916027665138, + "trace_id": "8313195f3b0683b1f92a019d1894caa0", + "request_id": "cf080191-a54f-4f0d-8a82-0e16602de594" + }, + { + "utc": 1795102043.3639, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018182125000748783, + "trace_id": "947dfbb890379c7c94d7c8a9d2467d3c", + "request_id": "08e1c3d4-5f89-4c75-9811-97340db54821" + }, + { + "utc": 1795102043.364148, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.027834666951093823, + "trace_id": "5f5f9b0a1d32bcfc1ee2c82a4b74d10e", + "request_id": "f48161e2-095d-4a48-aa76-af4fd5f9f09b" + }, + { + "utc": 1795102043.8685818, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006222125026397407, + "trace_id": "4898e6dbbbbc0ee615f260c31366457d", + "request_id": "90183ac9-5683-4c29-baba-799fe3bd2759" + }, + { + "utc": 1795102044.37215, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01759550004499033, + "trace_id": "decb96debe20112bb4cf30b1defdd3d4", + "request_id": "9fefd680-97f9-40ab-b9e5-15a595ce6e95" + }, + { + "utc": 1795102044.873868, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.014913625025656074, + "trace_id": "1846f57db783a4e71a515a4b1fc670bc", + "request_id": "a3378aff-0b9f-4360-904b-fc181b67419b" + }, + { + "utc": 1795102045.373905, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.019753667002078146, + "trace_id": "7ee10853a939294db426df830c14cec1", + "request_id": "ed921720-16b5-4fc8-8fde-43b4925b9b05" + }, + { + "utc": 1795102045.3739362, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03260116704041138, + "trace_id": "29cd28ba7a13708ff803e05e0bec3502", + "request_id": "5eaacb47-167b-47a1-a64d-c61105f4ddee" + }, + { + "utc": 1795102045.870146, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006258499983232468, + "trace_id": "d3e692aa440ba4e247297aa2bb301ea7", + "request_id": "44edc7cc-07a3-443a-814c-2135a456e631" + }, + { + "utc": 1795102046.369469, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018426167021971196, + "trace_id": "fafe7da224f9056232985a3228a7d400", + "request_id": "6b96a18e-15f8-422f-9e4d-946f463a7430" + }, + { + "utc": 1795102046.870153, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.009023666963912547, + "trace_id": "3e99caad3316b815e9b6b4d990d41be1", + "request_id": "58a843b9-a663-4f84-8e00-085b84702383" + }, + { + "utc": 1795102047.3712509, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017201125039719045, + "trace_id": "bc62d8399aa940ecc8fb75e56ba13dda", + "request_id": "fd6d44d8-517c-4a4f-9cf0-5b7188fc4f6e" + }, + { + "utc": 1795102047.371291, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028451332997065037, + "trace_id": "431dc01bc0fd6e0b6405554116e4eb94", + "request_id": "4a894e8d-3e87-44a6-9879-b3f78be23693" + }, + { + "utc": 1795102047.873823, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.0062419589958153665, + "trace_id": "e99b1d360d8276bf5ea0ec99c6b6e655", + "request_id": "98c4622f-cfad-4ae4-863d-2f3f36bb0955" + }, + { + "utc": 1795102048.372111, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01476387499133125, + "trace_id": "22778ea50636047d94809f14226c95cf", + "request_id": "ba05e2ab-8ed1-4c5c-b1f6-22999121977e" + }, + { + "utc": 1795102048.8650482, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.0057587920455262065, + "trace_id": "e2d4f8c2c9a18d09ab693c005d2c5563", + "request_id": "db1c3f2c-d2e3-4a0c-aa8b-f290ae3f5081" + }, + { + "utc": 1795102049.370178, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01929958298569545, + "trace_id": "42d105ad6fbb2660755b19c570d4645f", + "request_id": "1d1eccfb-6695-4814-8ab1-e6d70b1e2d6e" + }, + { + "utc": 1795102049.370218, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030939207994379103, + "trace_id": "613be45b193d4af353b57d4f6ca911dd", + "request_id": "544883ff-520a-4deb-a3b1-bb49fb73a7e7" + }, + { + "utc": 1795102049.873846, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005525209009647369, + "trace_id": "e2366c798feef8e759a7d908cdb21c37", + "request_id": "826e026d-de12-491f-b680-a028f9612c09" + }, + { + "utc": 1795102050.369386, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01472220802679658, + "trace_id": "9c2c6c0207772090e0cc57059d1c4f16", + "request_id": "18f89952-08b9-4c19-8d0c-424695ea0c9c" + }, + { + "utc": 1795102050.86574, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.014745875028893352, + "trace_id": "add17fa884775c7811f0dd89ce16b99a", + "request_id": "88cd8173-837c-4f9a-b47c-4bdbf884ef7e" + }, + { + "utc": 1795102051.371176, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018402084009721875, + "trace_id": "c7519d7227a6bdafed7379798bee234f", + "request_id": "02380b6a-8942-40d9-a5cc-d69b46868a84" + }, + { + "utc": 1795102051.371221, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030162416980601847, + "trace_id": "e5c5de65c9702dde340cff90f4891eb2", + "request_id": "b8f87b6f-6adc-4d8a-9936-b0074e9c4c02" + }, + { + "utc": 1795102051.87009, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.008066250011324883, + "trace_id": "7b144e454b0739ee70347b7fa79e7a0a", + "request_id": "22d86bdc-979a-45df-befe-b8773f631faf" + }, + { + "utc": 1795102052.366236, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017191208025906235, + "trace_id": "674534cf99e67e736a0b2e4e9db52cfa", + "request_id": "ab51d223-630e-4571-92e4-29001883624d" + }, + { + "utc": 1795102052.870124, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017897665966302156, + "trace_id": "d4decd1f5cc14a76497da152f737e2b5", + "request_id": "70be44dd-7f58-4bbb-984c-3c0f345832f0" + }, + { + "utc": 1795102053.370147, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.010482082958333194, + "trace_id": "0195a1c1dfc31321278c42d1b626d2dd", + "request_id": "2bfa08f4-88f7-41bd-94ff-8b724eeba543" + }, + { + "utc": 1795102053.370189, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01703037496190518, + "trace_id": "75f02fead42869c0678245b7c2fec33e", + "request_id": "ca27b2f3-c540-4f96-b1dd-ecba14709f3c" + }, + { + "utc": 1795102053.866094, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005329874984454364, + "trace_id": "bb20b6bf08f9e5aea8192dfbaa937f6c", + "request_id": "6958fdbe-8702-4e29-9b64-6334e1a0940b" + }, + { + "utc": 1795102054.373908, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016734457982238382, + "trace_id": "5e67ec101e5a4066523c9ef180a99fe5", + "request_id": "b0de2dbb-e60e-457b-a0ac-1e734e99a611" + }, + { + "utc": 1795102054.863931, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017673291033133864, + "trace_id": "6bb66aba96e4bade85225f04fdd5c6e7", + "request_id": "6f2ca48c-f400-46a4-bd12-422c6acde618" + }, + { + "utc": 1795102055.366109, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.00643754203338176, + "trace_id": "ae137ca8f44987e393b5657c5d94e9a1", + "request_id": "e58c3b02-a939-4dca-b52b-f2674a0bba4d" + }, + { + "utc": 1795102055.3661282, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011677332979161292, + "trace_id": "3f4849f105d8b6c6c7c64c89f17e2b1c", + "request_id": "87363940-ec9c-465f-9279-68677e9673f1" + }, + { + "utc": 1795102055.8663092, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005859250028152019, + "trace_id": "5a4f2c85d0a23d54778d3dc93dfa5260", + "request_id": "61eed47d-963b-4152-87dd-31eda9b8a189" + }, + { + "utc": 1795102056.36883, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.013553500000853091, + "trace_id": "6025d2af3c8ce5d5d9cf5752559ffbe2", + "request_id": "45eb9299-020d-4393-ad3d-823eed901b2f" + }, + { + "utc": 1795102056.867014, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01634720800211653, + "trace_id": "4deec8d01f677e00d11b994549aaf732", + "request_id": "12d3cdc8-94ec-4ac4-b35c-d806d8c04438" + }, + { + "utc": 1795102057.372048, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01813812501495704, + "trace_id": "ee0bd307d2b5324a68cb79dd9d62ca58", + "request_id": "162278a5-1ff8-4272-ac5f-173f35fd0f4e" + }, + { + "utc": 1795102057.3720891, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03363970801001415, + "trace_id": "7aa68e4cf9f16535a427c3f964ba8a52", + "request_id": "e11e54e2-5d63-4601-bc64-2a4273a5e8b7" + }, + { + "utc": 1795102057.864917, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006582917005289346, + "trace_id": "228e6a2182a662e752314c1cc904a08c", + "request_id": "13a29fd7-6f9a-4d12-b12c-4f7efb134a52" + }, + { + "utc": 1795102058.373906, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.009119666006881744, + "trace_id": "49fb5fd8afffbcf3877cf258e8019e3b", + "request_id": "9f65f33e-13bd-494d-b37c-e3afaef01be3" + }, + { + "utc": 1795102058.866202, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016445999965071678, + "trace_id": "60e7b214c8a29887ac049def94f9143b", + "request_id": "a0fa36f4-b357-4759-aaa7-ed6be35b68d0" + }, + { + "utc": 1795102059.3710458, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017571792006492615, + "trace_id": "73cd736de119bff9ad15a74bfec99774", + "request_id": "3bdf5f03-8e28-44f3-9332-bd3fab4e4c23" + }, + { + "utc": 1795102059.371084, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030451417027506977, + "trace_id": "b6d931116ee8e80e8f1cb7286ef5cf11", + "request_id": "9874ba8c-4c6c-415c-937a-840c6a56ad20" + }, + { + "utc": 1795102059.866125, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006671792012639344, + "trace_id": "59a88b7fa2674ee7098282c7febd8ff8", + "request_id": "1d1db0e3-bc68-4c48-8f55-35932715a202" + }, + { + "utc": 1795102060.3677208, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006652708980254829, + "trace_id": "0eea8b3d258486973d81552e2f6b7da8", + "request_id": "e868f55b-08e7-42fb-bc4f-9ed8e8ef85b3" + }, + { + "utc": 1795102060.87011, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01023287494899705, + "trace_id": "d74edf0378fd2f4db950729a6b0a266d", + "request_id": "4aa4edc1-286f-4189-8442-419ed27e8685" + }, + { + "utc": 1795102061.365367, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018712458026129752, + "trace_id": "9120deea9f976c7a3e2e2028b8666142", + "request_id": "9065c25a-1255-4a85-bbbe-1d57344ed5fd" + }, + { + "utc": 1795102061.3654022, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029276166984345764, + "trace_id": "43ec0cc83d9d0a161efdd2f05000de45", + "request_id": "e802d1e7-28ab-4837-a7bf-72d9270b9f6d" + }, + { + "utc": 1795102061.8700302, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005664833006449044, + "trace_id": "31b8307278397c2424edb02f16f8f226", + "request_id": "022eda65-b584-4903-8892-1a4ec2b5ee38" + }, + { + "utc": 1795102062.37009, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006324124988168478, + "trace_id": "416d2eebc21fa330d2b9d90370dd2c1d", + "request_id": "db6eb25b-ef58-4f9d-bc3b-7a071f978475" + }, + { + "utc": 1795102062.870157, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016687666007783264, + "trace_id": "7d52e0db2a434e7ee9a63ca65f337286", + "request_id": "26b17251-ed4e-4271-b500-c27beb504a92" + }, + { + "utc": 1795102063.373894, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018305958015844226, + "trace_id": "49b0ad07a74afc3a06f52d0794402744", + "request_id": "c69aabe3-a64c-4c54-8d09-a9e284f6aebc" + }, + { + "utc": 1795102063.373934, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030963332974351943, + "trace_id": "7df8f0225f96ec16ae9f1428ac1c411f", + "request_id": "d6572329-5548-4dbf-8898-1d97cb78f3dd" + }, + { + "utc": 1795102063.865835, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006611041026189923, + "trace_id": "aef691ea0ebae6113efc39565786e704", + "request_id": "2e4bb1ed-6c31-4774-a31b-28295eaee399" + }, + { + "utc": 1795102064.372792, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.025668125017546117, + "trace_id": "50d75fa7d6d6e364a09bda526e0fdf5c", + "request_id": "d14cc029-a042-41a3-92c4-f847c4967ede" + }, + { + "utc": 1795102064.8701, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017314125027041882, + "trace_id": "045d5b090000b9882cc3505cc4f0e765", + "request_id": "eb72f88f-290f-4de0-b0d1-40458f87501d" + }, + { + "utc": 1795102065.370083, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01896879100240767, + "trace_id": "93421498521eb1d3b41ef4fa2a84360b", + "request_id": "28a98708-ed10-4315-b1b4-2e1e5229d9af" + }, + { + "utc": 1795102065.3701298, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03143595898291096, + "trace_id": "573459519c6620fbdfd7f9a1ccd6992c", + "request_id": "412e6fe5-26d5-4853-be18-df58c2e2411d" + }, + { + "utc": 1795102065.870019, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005080333969090134, + "trace_id": "9a0f63162579d1eb8d49f1fa0649c95e", + "request_id": "5fe39ed3-a473-4b6c-a173-f19f5012d9dc" + }, + { + "utc": 1795102066.366165, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.015342833998147398, + "trace_id": "3d405bba7792ae8003ec4b77dbd3e844", + "request_id": "bf24768d-5784-4ae4-9fb2-2d5f9ae3814b" + }, + { + "utc": 1795102066.870074, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01646516699111089, + "trace_id": "44e7355c8cff1945052bbf80b082eba9", + "request_id": "0ea8b707-7f42-4d17-a390-8e85a7daf2a8" + }, + { + "utc": 1795102067.367797, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01687087502796203, + "trace_id": "6305c20faf54fb019bf7d08504282a14", + "request_id": "b88a5434-8933-410f-be59-f0cf8d070cca" + }, + { + "utc": 1795102067.367837, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029764666978735477, + "trace_id": "b55b5cb08b4059c7be719651fd91e030", + "request_id": "fd84ee2b-0d8a-4680-9e92-d384e6e2aa55" + }, + { + "utc": 1795102067.870024, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.0056434579892084, + "trace_id": "9d8b99e0243fb8b1a94360c05031ffff", + "request_id": "ea8359dd-304f-48ec-b12d-94374f4d354f" + }, + { + "utc": 1795102068.373871, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.007988500001374632, + "trace_id": "d4ae4b11b15b37574e38d0481ca5df0c", + "request_id": "f9c09e86-c4f0-49ef-9019-1eb9c3b8b263" + }, + { + "utc": 1795102068.8738592, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01868245896184817, + "trace_id": "7b87d3dbc892bd19c4ff0d816a67fea5", + "request_id": "fd815de1-f443-4771-95e6-af6625fdbbff" + }, + { + "utc": 1795102069.364944, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016268958046566695, + "trace_id": "c626effca5adfd173d820139638cab00", + "request_id": "7d9c14a1-bdcb-483b-a398-10f048138ad2" + }, + { + "utc": 1795102069.364971, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028602125006727874, + "trace_id": "3a6b5d417cb1d636d0803cb59f3a17bb", + "request_id": "e9086cda-5716-4ebe-9398-d90fdfa5ad32" + }, + { + "utc": 1795102069.87379, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006768958002794534, + "trace_id": "9975289a371e8714cbe5333a891cbb04", + "request_id": "9fd659da-8389-42d0-9cac-72383ba84dee" + }, + { + "utc": 1795102070.3737679, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005787792033515871, + "trace_id": "81d7d7614624e99877c0b3d793e25da3", + "request_id": "13a9b495-d778-4b62-9406-25f012afe4e2" + }, + { + "utc": 1795102070.870986, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01585066702682525, + "trace_id": "83d0c45b1c1c79aec22f99036254b065", + "request_id": "d0c7d713-8e9b-4eaf-8ec1-e3f270fe64f9" + }, + { + "utc": 1795102071.371179, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018464084016159177, + "trace_id": "abcbd01336f987570f905fdc5cb61b16", + "request_id": "e33a0928-d78f-44dd-883f-43e8b21a405d" + }, + { + "utc": 1795102071.371217, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03105145803419873, + "trace_id": "8211181f2aaaaef24a124f8e9dd317d4", + "request_id": "8bad6173-3d92-46e7-80a1-cc13de0e05a1" + }, + { + "utc": 1795102071.873913, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.010828875005245209, + "trace_id": "3bcf82b28a6232d3f1ae8a6938ed031e", + "request_id": "7706be23-920e-47ce-9613-b25389b5b225" + }, + { + "utc": 1795102072.373863, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017367125023156404, + "trace_id": "e0535740e621c222fd565f02e83f0e4e", + "request_id": "2c9157d0-6bc2-4972-8c75-112121ca642c" + }, + { + "utc": 1795102072.872023, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018824416038114578, + "trace_id": "0b732b7310009b8df6fc260f877e4221", + "request_id": "adaae8ae-1afd-40b2-ad0d-002c75589f28" + }, + { + "utc": 1795102073.371988, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.019562667002901435, + "trace_id": "531eb6da1b48379924a5b12d6571cd5c", + "request_id": "0d95e2f6-8477-46b6-b0a5-471a38699594" + }, + { + "utc": 1795102073.372027, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030808374984189868, + "trace_id": "023e56b68064c4dbfbf502da39544694", + "request_id": "94d14de3-b041-42f7-a465-61f5dbd8b05c" + }, + { + "utc": 1795102073.871976, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.009088957973290235, + "trace_id": "7894d0603d84690a1b88d6d917706883", + "request_id": "a683eb84-1b90-44a7-8176-244a07625a0f" + }, + { + "utc": 1795102074.370521, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017414707981515676, + "trace_id": "566e86a9aba7dd670dee3ffb9664eccb", + "request_id": "3be21435-1c7d-4e48-9bc0-b675126b4227" + }, + { + "utc": 1795102074.8700511, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.008454333001282066, + "trace_id": "fb6177a054f9d4a3c49ee52e060b1971", + "request_id": "78e692a7-3159-4a0a-93d0-19983b3c3564" + }, + { + "utc": 1795102075.370028, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01711758301826194, + "trace_id": "dd3b41fd53ec3de888b199dec69ab4c5", + "request_id": "8c421930-c7e4-4787-98e8-0133362ce040" + }, + { + "utc": 1795102075.370052, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028519124956801534, + "trace_id": "c6b5a876ede79ad9cfd5e2062412c00d", + "request_id": "35c29300-b61d-46ec-b05b-7c904dc5661a" + }, + { + "utc": 1795102075.870486, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.00494999997317791, + "trace_id": "9c9728706688622989bdabeb8001c289", + "request_id": "ccd54c16-06a1-4050-a5c4-5caab6adaff8" + }, + { + "utc": 1795102076.373894, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016104458016343415, + "trace_id": "91a6a4f322a27e841eee6b9bd41c453c", + "request_id": "20dc3172-6fa1-4e8f-b900-3918ec84dc3e" + }, + { + "utc": 1795102076.869954, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005842541984748095, + "trace_id": "486479d218fb546eeb38600446f8d709", + "request_id": "d7e64cda-41e4-4791-a61f-69e3e538c4ea" + }, + { + "utc": 1795102077.370053, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017268957977648824, + "trace_id": "f11fd782e7e07cec656a6c8038f69785", + "request_id": "66648c21-9d3f-4dd1-8b36-f7c813803d91" + }, + { + "utc": 1795102077.370092, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.027958207996562123, + "trace_id": "995e14f32b459c0d9eb6cf1c208600f6", + "request_id": "f1af0341-0f69-4e7b-afcc-291a8879df63" + }, + { + "utc": 1795102077.8718479, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006045249989256263, + "trace_id": "240cac98e41f76cf796a553df4b79a60", + "request_id": "ce0db78d-b70e-416f-ae3f-ec8b087f3bce" + }, + { + "utc": 1795102078.3719628, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01611029199557379, + "trace_id": "687d153d5cdfdd627dfc0096bb3fcff0", + "request_id": "7f47bc2d-24ab-4b6d-a95e-b41eded99007" + }, + { + "utc": 1795102078.871518, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.015039499965496361, + "trace_id": "8ebdb5fde9db95f7125ddeeea9776343", + "request_id": "0bca23a1-1c7e-48c4-b643-249a0f54ff17" + }, + { + "utc": 1795102079.371349, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01778116700006649, + "trace_id": "cf33b2839deeeb33069155a869127528", + "request_id": "52f6df34-2fde-4471-bda9-bedeb280c94f" + }, + { + "utc": 1795102079.371391, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03018870804226026, + "trace_id": "ff2e9584f3093ed87e87fea90dc1b108", + "request_id": "0e18ee5b-d2ec-40a8-ba02-7dc819e9e0d9" + }, + { + "utc": 1795102079.871834, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005053624976426363, + "trace_id": "703ca8aacb160ab87ac9893eb3de8d19", + "request_id": "c9e58f00-e437-4e30-a796-a20661bbeacf" + }, + { + "utc": 1795102080.3705668, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018080458976328373, + "trace_id": "0baed7f23e58cc0b605da8315da51cd2", + "request_id": "693b39dd-55a1-4e84-9e2b-feab51070741" + }, + { + "utc": 1795102080.8702788, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01734400005079806, + "trace_id": "dee9ed8250369a216a12967b9cc8e5c1", + "request_id": "6e11eea8-5b13-4f06-9d15-6456d00549da" + }, + { + "utc": 1795102081.371945, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.010672500007785857, + "trace_id": "07b43983a4dddae4d707040400ff06f7", + "request_id": "33325bfd-d90e-4f6f-bc5f-e9c17b4b1315" + }, + { + "utc": 1795102081.371986, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01955045899376273, + "trace_id": "00e28ab79d4eae540ffc3ba0b3adde9c", + "request_id": "c163d12c-90ad-4aa9-9640-0a20ba7dc2bb" + }, + { + "utc": 1795102081.8682182, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006923332985024899, + "trace_id": "75a6c4bf964e3e6b062f2ea7650e42a3", + "request_id": "faaf6ba4-762c-4ae5-94b4-11719a2f9986" + }, + { + "utc": 1795102082.370025, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.017392624984495342, + "trace_id": "24aae8f6ec0ca1a06b662ef26321cbb2", + "request_id": "ae03ec0b-b425-47bf-908a-58a89c2b8453" + }, + { + "utc": 1795102082.871892, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.014474458992481232, + "trace_id": "73f260ecc3bf23b90e89cd0b67ce2308", + "request_id": "e9816152-4797-47c2-9972-476f8dc85a93" + }, + { + "utc": 1795102083.373853, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.006752000015694648, + "trace_id": "468a29c0116925399b3a44eee8d0954a", + "request_id": "a0ce3cd5-43bf-40ae-82b3-e1580160f8e6" + }, + { + "utc": 1795102083.373888, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011470375000499189, + "trace_id": "38791b45dc5bbbfb8d7996819d441c38", + "request_id": "b8a1b38d-5e9d-4747-af77-ac75d63ce4ea" + }, + { + "utc": 1795102083.86433, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005336874979548156, + "trace_id": "d1643aaee795da3ba107d090fb225b0c", + "request_id": "e0452223-5f53-41cc-aef2-c55ab9623687" + }, + { + "utc": 1795102084.368781, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01495641702786088, + "trace_id": "b77d1d30f17676e2a5237aca2cb2af4d", + "request_id": "f4dfe372-740b-47e9-9f6c-2f7de780772b" + }, + { + "utc": 1795102084.8719301, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.009630999993532896, + "trace_id": "4afe7e78b4a7a4a047044b865a049f5f", + "request_id": "73d430dd-7198-42b2-95a1-20e24406fd88" + }, + { + "utc": 1795102085.364589, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.021705708000808954, + "trace_id": "976344c73978fb4d17d7ed8473609be7", + "request_id": "c005e5bf-c49d-458d-bc7d-22fb85507044" + }, + { + "utc": 1795102085.3646688, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.031577665999066085, + "trace_id": "a027993cc0d70a161b4e7e4854cc7379", + "request_id": "8ffcb267-10a6-4100-a902-7a84e0c35fdd" + }, + { + "utc": 1795102085.8719418, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.007941041956655681, + "trace_id": "2a1a43c0a55b71fb979ebe2d80eb68d3", + "request_id": "b4a5dc63-c044-4620-9849-1279a7f87b17" + }, + { + "utc": 1795102086.371917, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.015304375032428652, + "trace_id": "2a45b499e762933eaa6c44d134f16aa0", + "request_id": "046398f3-748c-4ce3-81bd-ed9ecf809ae1" + }, + { + "utc": 1795102086.871962, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.008994999981950969, + "trace_id": "8d3e8e4e010a95d1251fcb554f3a4eb3", + "request_id": "ce225535-e0f1-4263-bc4d-57d3d54f59f8" + }, + { + "utc": 1795102087.370692, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016668374999426305, + "trace_id": "25db2fc1267b138975457bd5da54a7ea", + "request_id": "85fe1edf-5235-4ac1-be7e-7ae14621c7a7" + }, + { + "utc": 1795102087.370734, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02794729103334248, + "trace_id": "beb5f986a8141c88c91f49e406b3dc0e", + "request_id": "284af999-bb2a-46fd-9ece-2abedcacd148" + }, + { + "utc": 1795102087.870305, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.00757566699758172, + "trace_id": "c8fa3c581423cbbce40eecb7618bb61d", + "request_id": "76e68dd6-a717-4a3f-b10f-ebc50fbe9f0c" + }, + { + "utc": 1795102088.37153, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.016646417032461613, + "trace_id": "361077a03a651f7a2432a334584a6baf", + "request_id": "cc2b7805-cab1-40d6-9681-b2db49169326" + }, + { + "utc": 1795102088.8657658, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005648999998811632, + "trace_id": "d20531b0fa5d49a657939bc85119d2aa", + "request_id": "228d0cb9-c8b8-4e38-a12b-8b6d1882696e" + }, + { + "utc": 1795102089.370032, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.02028095902642235, + "trace_id": "d5bac53de89f149a71066f46be688ead", + "request_id": "d5df5151-e756-4ef8-bb4b-4758f397aa6c" + }, + { + "utc": 1795102089.370522, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030415291956160218, + "trace_id": "e7c41774921acc827f51aee26fba6cfc", + "request_id": "754d8134-3aee-458f-99e8-72debf0f0e5b" + }, + { + "utc": 1795102089.871827, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005369040998630226, + "trace_id": "cd6c4f221225d6aad7bde61b3bb655fa", + "request_id": "25de55c2-6a22-4192-8fe1-6b2c44a4a870" + }, + { + "utc": 1795102090.369954, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.015973082976415753, + "trace_id": "036cb7506d5d9ad1c9c1ba40a46f09a9", + "request_id": "90925f25-f069-4e92-a05a-f4cc3f6e4dd7" + }, + { + "utc": 1795102090.8705559, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.011246833018958569, + "trace_id": "dfe6aad002408c0e87b1faafa9bee656", + "request_id": "6494295f-844f-4ee8-a1e8-15967757a052" + }, + { + "utc": 1795102091.364833, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01657512498786673, + "trace_id": "41b26cffb5fda17b7b6dec371ba40c43", + "request_id": "81a55945-8e23-4483-ab75-0cd4309e258e" + }, + { + "utc": 1795102091.3648589, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.0255113749881275, + "trace_id": "cd464d0476571004175034ebe4590001", + "request_id": "29174995-c34c-4958-8538-3b97ab8a874b" + }, + { + "utc": 1795102091.873701, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.005701999994926155, + "trace_id": "78594dd3508509f1c03c5028ed15330f", + "request_id": "751f584a-2b60-45c4-b6d6-27a65c6fd6cc" + }, + { + "utc": 1795102092.366403, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01059554098173976, + "trace_id": "a80131e85816a03004ef0e03af709696", + "request_id": "0742449e-be10-4b8a-a6e3-5023ae7dc449" + }, + { + "utc": 1795102092.8718479, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.014767417043913156, + "trace_id": "954d1ed6a206a9a159b3fdee2313e45f", + "request_id": "d353c0f1-1036-4c14-ae38-d4ee66e31056" + }, + { + "utc": 1795102093.3738089, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01907683297758922, + "trace_id": "55f22032996b29aebe1d4f5b92ee5999", + "request_id": "bbd66704-0613-4361-b926-f3533f00817c" + }, + { + "utc": 1795102093.373844, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029774166003335267, + "trace_id": "8e88d4223d9ccb18dcd535f472616854", + "request_id": "8a2f186b-4168-48ab-bcd4-d8998cf320f3" + }, + { + "utc": 1795102093.869997, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.00673566700424999, + "trace_id": "09764f72e5fae2f2ceecd620bf6dcea3", + "request_id": "803182d3-a32e-42a6-a2a7-17f1af0d74d5" + }, + { + "utc": 1795102094.366011, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.01644612499512732, + "trace_id": "c20b607319d5d40cd41f41c960a8dd4b", + "request_id": "d8af183d-a624-48ef-8ae8-4b54bfed552a" + }, + { + "utc": 1795102094.8718789, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.014840457995887846, + "trace_id": "6cc2acc564c020db383723bf93db12bd", + "request_id": "69157fde-15e2-4638-a676-f7b96dd23ddc" + }, + { + "utc": 1795102095.369975, + "phase": "baseline", + "kind": "message", + "status": 201, + "duration_seconds": 0.018914040992967784, + "trace_id": "1a1c2567e3f75aa4bd4810aed379cf64", + "request_id": "afb7cf03-3b37-48ac-a93e-d59187c097c5" + }, + { + "utc": 1795102095.370015, + "phase": "baseline", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03519266698276624, + "trace_id": "52afeab2470cd5ed7fc59f7b106a944d", + "request_id": "26c4f2d4-8a51-4155-b4a8-fe592901e12d" + }, + { + "utc": 1795102097.364834, + "phase": "short_db_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03150966699467972, + "trace_id": "c011b916203bde986cb3e52fd93537f3", + "request_id": "fef948a1-e5a3-4a62-941e-5f5c63cc5130" + }, + { + "utc": 1795102095.8719192, + "phase": "short_db_spike", + "kind": "message", + "status": 503, + "duration_seconds": 2.0082989169750363, + "trace_id": "68451a79a885af8177d6c03514b1c52a", + "request_id": "15a744c4-2b14-405f-99b8-b83254946075" + }, + { + "utc": 1795102096.366003, + "phase": "short_db_spike", + "kind": "message", + "status": 503, + "duration_seconds": 2.007859290984925, + "trace_id": "33864c9b1fecf2ad6736dc8c44a28a88", + "request_id": "845dc8ab-8a8c-4d79-be64-d185c3446246" + }, + { + "utc": 1795102096.866086, + "phase": "short_db_spike", + "kind": "message", + "status": 503, + "duration_seconds": 2.0125928340130486, + "trace_id": "633b084c0aa596cc42cb1583636ee6d2", + "request_id": "7c9bd8fb-f540-4d2c-b50f-230d0d9d4b79" + }, + { + "utc": 1795102097.364683, + "phase": "short_db_spike", + "kind": "message", + "status": 503, + "duration_seconds": 2.0171876669628546, + "trace_id": "dc883eb261b1259a503c50515afa7a69", + "request_id": "11744bc0-825e-4435-a4b8-a5b5f7ae2c4b" + }, + { + "utc": 1795102099.364935, + "phase": "short_db_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03169362503103912, + "trace_id": "8c10fe6689cbfee092130ffcfd29e6ed", + "request_id": "c504d5a0-53a8-4119-b1eb-6ae05a260407" + }, + { + "utc": 1795102098.3716218, + "phase": "short_db_spike", + "kind": "message", + "status": 201, + "duration_seconds": 1.1684529170161113, + "trace_id": "510bbd5975b0f641a93f481bf2f4adf4", + "request_id": "d278dac5-6d97-4301-ba7f-432905123a7e" + }, + { + "utc": 1795102098.866996, + "phase": "short_db_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.6760163750150241, + "trace_id": "a4a15cd6f3de8730833c34f440b7ec91", + "request_id": "b2788ff1-4b90-41f2-9e00-63528a08bde0" + }, + { + "utc": 1795102099.364894, + "phase": "short_db_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.17898041696753353, + "trace_id": "3130b19b9b05ed53f70bda63f1d79d41", + "request_id": "5137c819-9ec0-4b2f-b6f2-07b3b3c4064c" + }, + { + "utc": 1795102097.873674, + "phase": "short_db_spike", + "kind": "message", + "status": 201, + "duration_seconds": 1.6707140830112621, + "trace_id": "01116edebace8cffa1bf47865a5f2bb4", + "request_id": "c454c3e3-d720-4dff-9b4c-16a61d811fa0" + }, + { + "utc": 1795102099.865547, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.00682258396409452, + "trace_id": "a0c04602df63463dc818ccce6a172fed", + "request_id": "2e8dc799-3f20-45c8-a3ab-dadde84fb8fb" + }, + { + "utc": 1795102100.370483, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.0200247920001857, + "trace_id": "a89b2bfb721a4487fd3d88ec49cab90a", + "request_id": "7a4ea998-4800-425c-8d7d-bd4ae29cead1" + }, + { + "utc": 1795102100.873309, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01810133398976177, + "trace_id": "583dd0fe903158fc665476b33c9e5fb6", + "request_id": "28d32581-40a1-436c-8fb9-6755a6dc9815" + }, + { + "utc": 1795102101.369982, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.017316124984063208, + "trace_id": "36883f3bffc77662b07f261d6c6d7a2e", + "request_id": "9ab1f450-0025-4c04-aa36-7af07d69afa1" + }, + { + "utc": 1795102101.3700302, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029834333050530404, + "trace_id": "b7cd619a1ade235f04db5b8c135db019", + "request_id": "51edb81d-8dca-4964-9557-9af1977b9380" + }, + { + "utc": 1795102101.866116, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.00907945801736787, + "trace_id": "6c717fe9934f23cf9daff0f106f7d232", + "request_id": "06e97071-8fdf-4525-b9ff-c8256feba8f9" + }, + { + "utc": 1795102102.3663821, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.017930875008460134, + "trace_id": "328dd5acaf6d4677a5cbf689568cbae7", + "request_id": "f052e03a-8fa7-448d-af11-06ca9e8832b5" + }, + { + "utc": 1795102102.869986, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.00902687496272847, + "trace_id": "a2c5b69babc5c5260301b16cec5c28d0", + "request_id": "f7850138-cfad-40ed-ad41-5fc26ee25b6b" + }, + { + "utc": 1795102103.373785, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.020131209050305188, + "trace_id": "9340b250d02f9333d74e8591c0cceb40", + "request_id": "8ef566c1-4e9f-4dc1-b52c-7a84b9410ebd" + }, + { + "utc": 1795102103.373832, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03208991599967703, + "trace_id": "d5792b8ac029936569692c3577e44c77", + "request_id": "aa5741c9-189f-4612-b33f-25efbb9620de" + }, + { + "utc": 1795102103.8737228, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.008235958986915648, + "trace_id": "088f018be6e8a31b770440bd577e0a66", + "request_id": "f7bb3670-7854-422a-8cfd-76e12f638043" + }, + { + "utc": 1795102104.373739, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.0168477920233272, + "trace_id": "b0d405bf07fe9e51202e228e83db82c2", + "request_id": "a2511f5c-ca9c-450e-b5bc-cd65cc8e20b6" + }, + { + "utc": 1795102104.863817, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.005870459019206464, + "trace_id": "000197cdf49582422d95cbb827e1692a", + "request_id": "7c368ece-c6f7-4180-acdc-45efc13e12f1" + }, + { + "utc": 1795102105.372202, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01918966695666313, + "trace_id": "f6603e9f03084cff8eaaab6de9744efa", + "request_id": "d4380803-c961-436c-b08e-76cb36f683f8" + }, + { + "utc": 1795102105.372246, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030739333014935255, + "trace_id": "2d7f93d8fcaacac0c0678509b2177515", + "request_id": "072e374e-a8f1-4700-93b5-e68085b0e179" + }, + { + "utc": 1795102105.871874, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.005411541962530464, + "trace_id": "75c31ddf134545a7e6f593c87dd2c99d", + "request_id": "210f8ed3-25ad-409b-a7fa-9054ad2f24a2" + }, + { + "utc": 1795102106.3720372, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.017557542014401406, + "trace_id": "fc32cdaa09574240ac4b9acf9a6a83cb", + "request_id": "187d178b-4479-41bc-a8ed-d56f2c919be3" + }, + { + "utc": 1795102106.869932, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.018274249974638224, + "trace_id": "9f1e2946c92bc489a67d9127201255d7", + "request_id": "28fe0cc0-739c-4533-be60-dc55f5ecd22a" + }, + { + "utc": 1795102107.3737652, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01146770897321403, + "trace_id": "d9f4ae875064d7dfb9a4d6ee7b06392a", + "request_id": "4bd71a4d-e6cd-4a54-b581-257b98aa3201" + }, + { + "utc": 1795102107.373811, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.019796375010628253, + "trace_id": "60fd16e4a8232091965743334e55d10e", + "request_id": "118ed6b2-6f91-4353-9bed-edee0f064aaf" + }, + { + "utc": 1795102107.8698611, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.005843874998390675, + "trace_id": "be11ba6606ea67917931fd6fb0155d65", + "request_id": "0017579f-f854-4623-a77f-dfd9849170a9" + }, + { + "utc": 1795102108.36638, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.018849999993108213, + "trace_id": "dd1607acde00c398b37e8b2ce155584c", + "request_id": "ac7f8319-279e-46be-80d9-a529a2ee0b8d" + }, + { + "utc": 1795102108.873711, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.017615624994505197, + "trace_id": "1c182e249ae94980fc2018b04e656297", + "request_id": "b6396bda-49f1-4291-86c1-54c325209fe5" + }, + { + "utc": 1795102109.364736, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.007442916976287961, + "trace_id": "bac6953f16b6eb529215fc8f15c3de78", + "request_id": "0a6724aa-3f52-48db-be7e-4e4a9d26cf6c" + }, + { + "utc": 1795102109.364782, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.013441291986964643, + "trace_id": "632cda886a6a39ea81e67b91019985ef", + "request_id": "61d7bf10-8112-4e61-aa6e-39cb316fc536" + }, + { + "utc": 1795102109.871742, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.005096625012811273, + "trace_id": "fdac0bb55c4bc2bd26490f7a5b7050e0", + "request_id": "7a65e02c-801d-49fd-892e-e00857797ce6" + }, + { + "utc": 1795102110.371822, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.016509625012986362, + "trace_id": "c844ec00a967b2e856bfb1329de6a349", + "request_id": "65a9f86b-7d46-4e14-adfe-e2dd41369e0a" + }, + { + "utc": 1795102110.869925, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.016475000011269003, + "trace_id": "3d8f8a9188b3f5c31c2dafe532676cbd", + "request_id": "70c8713f-5f63-417e-80a9-e235250a2483" + }, + { + "utc": 1795102111.3736541, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.007915791007690132, + "trace_id": "4bf27559ff1d0f40c1219481102b4c78", + "request_id": "c8f96584-a5af-4e6c-9c99-b46b5cd027de" + }, + { + "utc": 1795102111.373684, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.013563000014983118, + "trace_id": "2c04f94feec7db842043257128638b58", + "request_id": "4636eb63-e85e-4ae2-891e-4069893726c1" + }, + { + "utc": 1795102111.867836, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.010409000038634986, + "trace_id": "fc06d84ef89660ce0d9a72980e88e028", + "request_id": "d64636f1-8572-4d51-b47a-d8cd7894b0ef" + }, + { + "utc": 1795102112.366838, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01571558299474418, + "trace_id": "25d2e29ea7dd9e1fce855ed4022dc794", + "request_id": "55f3b97e-b5db-4e7c-b889-3e4f884e48d9" + }, + { + "utc": 1795102112.873734, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.017652041977271438, + "trace_id": "692c0d478b3f130997c2e623f87ceffb", + "request_id": "554a7864-8bf5-40f5-92f2-1136c6c6e035" + }, + { + "utc": 1795102113.36599, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.009068125043995678, + "trace_id": "39ca96490414280b2b408592a0017978", + "request_id": "ef320153-8cfc-4803-a69c-7f0f97a4d3ad" + }, + { + "utc": 1795102113.366028, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.014117707964032888, + "trace_id": "c8e287158adfd9130a28305eff3d5938", + "request_id": "9c720129-a872-4688-bc77-70b72e6d9cb8" + }, + { + "utc": 1795102113.8737102, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.0072717079892754555, + "trace_id": "66ce7631ac8a1eabf8af9e5b08edb234", + "request_id": "b1ae37ba-8d5b-4d06-9c01-19e499ea5195" + }, + { + "utc": 1795102114.371809, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.0178013329859823, + "trace_id": "37219b1546a6363ed948c57968bcffdc", + "request_id": "c53668dc-e20a-406a-b5d2-aff26730f674" + }, + { + "utc": 1795102114.868298, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01614058401901275, + "trace_id": "08f471fa9c16839c601ddb11664b8c24", + "request_id": "e86d35d5-aa7d-4f96-ac47-789de4c21520" + }, + { + "utc": 1795102115.365952, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.015058165998198092, + "trace_id": "cebc3faf77635727c0441be546e54072", + "request_id": "419870c4-2f9f-4446-89e9-ae1bfef369bc" + }, + { + "utc": 1795102115.365979, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02827183296903968, + "trace_id": "60d368f3a60f583173cb55f190975580", + "request_id": "c69241ad-e3b5-483c-a5a1-8d07f7f77ea2" + }, + { + "utc": 1795102115.873735, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.005452750017866492, + "trace_id": "b75fa35971a743693cef455edf6e5b6c", + "request_id": "89cd43d0-0b46-4846-bdbe-411c2377f996" + }, + { + "utc": 1795102116.3643572, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.018800667021423578, + "trace_id": "f49b846456aaeea0293d557e8f83fead", + "request_id": "c85d1db1-8c79-4e57-8f67-9c95097267aa" + }, + { + "utc": 1795102116.871839, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.019122208992484957, + "trace_id": "ea19383c159a4ca98509b90d790d06c8", + "request_id": "95ef94db-4cc2-4d47-866c-6151be387433" + }, + { + "utc": 1795102117.373791, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01965441700303927, + "trace_id": "ea3f9569e0003ac26ac34442a5de8e3c", + "request_id": "beb155d8-4843-405c-89ae-c6aebde532fd" + }, + { + "utc": 1795102117.373835, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03773495799396187, + "trace_id": "24ab7d7e8fd8ff0fe6a63062bf40058a", + "request_id": "35bb342a-1ee5-49b6-ad41-38cf139a7be8" + }, + { + "utc": 1795102117.866016, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.006029375013895333, + "trace_id": "cd463c0e6df3b8d3ca69e5cbc6e36e14", + "request_id": "b0f7b4c8-41ac-4ff1-a85b-673fcb24049c" + }, + { + "utc": 1795102118.371901, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.017652541981078684, + "trace_id": "d4eafdb3ece67fa4acdf085b9b6253fc", + "request_id": "01911c8b-ea96-4d34-b2af-5cdd99f02b74" + }, + { + "utc": 1795102118.871794, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01647495897486806, + "trace_id": "e88e2fca9a7b9f84e91c0b3363f9d580", + "request_id": "719a68f0-85a5-467f-b920-fc5d6e0440af" + }, + { + "utc": 1795102119.3717718, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.018383999995421618, + "trace_id": "9eba20852f9b929c1ad9575cc30d5744", + "request_id": "811eac1d-d9a9-45e3-b3a1-d650bd424d3d" + }, + { + "utc": 1795102119.3718028, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03157608298351988, + "trace_id": "988db3b9d6d8f0ad0a41f45cf75ed0b5", + "request_id": "69998a3c-6a81-46e6-9e65-d2d7bac092f9" + }, + { + "utc": 1795102119.8654761, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.006519334041513503, + "trace_id": "93dc95857f7580fb3be4eb81effc1a4e", + "request_id": "fb972a37-4e99-4f3f-8d02-c7d924526c86" + }, + { + "utc": 1795102120.364352, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.011971917003393173, + "trace_id": "f44cdeade4bf2d3c00289c8c22816227", + "request_id": "77711a64-3e52-462e-b783-ebbd632fc7a6" + }, + { + "utc": 1795102120.865643, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.015835374942980707, + "trace_id": "200d16ebb391340f27a4c96ad19cccb1", + "request_id": "148526e8-d615-4e2c-adec-7090e4f1b560" + }, + { + "utc": 1795102121.367203, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.016934917017351836, + "trace_id": "93ad9a4742849649fda5a1527440c0c9", + "request_id": "f317e77b-d202-43f6-9834-ce2f26dfbf68" + }, + { + "utc": 1795102121.3672478, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029498999996576458, + "trace_id": "f71bc3f112c153d80eda12c17ef80214", + "request_id": "f755dc9a-9b47-4d9c-88d7-60eedfe8d1a7" + }, + { + "utc": 1795102121.8704472, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.00860879203537479, + "trace_id": "37ab61696d658becf5fce61162db5c47", + "request_id": "54f195bc-10d5-42b8-9ee4-533b321b541e" + }, + { + "utc": 1795102122.366441, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01692558400100097, + "trace_id": "3ada847d418eed6d9547b5912fab9cc9", + "request_id": "15388a7b-3612-4c5c-9b2a-4b320c3de44a" + }, + { + "utc": 1795102122.8721778, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01669304200913757, + "trace_id": "e9a719470f1cf8d46bf75c17d5f0cdf3", + "request_id": "868f2b10-405b-40b1-b36a-50372c82cf3b" + }, + { + "utc": 1795102123.369866, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.018499458965379745, + "trace_id": "2db28327aee8ef03c541a3b04f4346d2", + "request_id": "934391e5-4239-4591-9b38-278678294fe8" + }, + { + "utc": 1795102123.3698938, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03046004200587049, + "trace_id": "d3c1cd5e883d2461ab394eaa3278caa4", + "request_id": "c82d2ba1-0e88-484b-9398-f1b4c3880ef9" + }, + { + "utc": 1795102123.873687, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.0073148750234395266, + "trace_id": "8b2895c4d366d2dc8540909204769867", + "request_id": "f572be7f-1718-4b46-80d0-2f8197ef7fb5" + }, + { + "utc": 1795102124.371779, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.006448875006753951, + "trace_id": "85a8f8d66b744c6e5f921e35d9fa1659", + "request_id": "97ddfda4-38a8-45ef-aa7e-70c0502a5cff" + }, + { + "utc": 1795102124.8736992, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.016222874983213842, + "trace_id": "61bcb5c172303e65d63cc7cf40c63b71", + "request_id": "95ae2d76-7fc1-4f15-9c38-c213fda68962" + }, + { + "utc": 1795102125.364755, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.019029000017326325, + "trace_id": "818eb80dc4e4bad978214ad6e59d1e72", + "request_id": "e5b97c30-00d7-4e26-ae11-d7ba0d2363fd" + }, + { + "utc": 1795102125.364794, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030234624980948865, + "trace_id": "6d94c8ee45ea8fa84592e83419324547", + "request_id": "c62ea3c1-8964-48ab-87cd-8ee9ff752e0b" + }, + { + "utc": 1795102125.869905, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.006597874977160245, + "trace_id": "6dc1446a717f00df1c953c4e284d1aed", + "request_id": "efdfd6de-ed56-4479-9f1d-80739e2a6d08" + }, + { + "utc": 1795102126.369737, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.007278707984369248, + "trace_id": "3e5cab2357206dc8cd07eaf202e474df", + "request_id": "10798dbc-0030-44ad-b808-0e404c596ab2" + }, + { + "utc": 1795102126.8715441, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.016591165971476585, + "trace_id": "c6f949f627f967b930bb401ce766b7dc", + "request_id": "abefad2d-47ee-4061-b1e6-c61f421cfbd6" + }, + { + "utc": 1795102127.369885, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.019952583010308444, + "trace_id": "3834b93a646e4b1b8843e2f3409a0c36", + "request_id": "bd31e8c1-8dd7-43e3-a60c-66ca0be2ce03" + }, + { + "utc": 1795102127.369928, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03025995899224654, + "trace_id": "b79bcd64c70ab1a5804f34729b51fe21", + "request_id": "84568710-1da8-42e2-bf6a-6469b4a9862f" + }, + { + "utc": 1795102127.871677, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.005741125030908734, + "trace_id": "4a8a4f1264f6bd460525eb8f5f2b2488", + "request_id": "fcae580d-de9b-4919-9de8-92cfa745e1b5" + }, + { + "utc": 1795102128.3698711, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.014727292000316083, + "trace_id": "0b2ee6d05b6782acd8d6c3871aa1b8c9", + "request_id": "f7f27885-0f34-4ede-ac98-d63d5d31118b" + }, + { + "utc": 1795102128.868395, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01567095803329721, + "trace_id": "e85a0d9142e2572db46bd5516538104f", + "request_id": "8620bab0-72ae-403e-a48c-6a841e50fb58" + }, + { + "utc": 1795102129.36786, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.017515957995783538, + "trace_id": "44c1c3131d03b067e085f9c7e016b28d", + "request_id": "c20f72ff-89cd-4625-96d7-25207fb15bd3" + }, + { + "utc": 1795102129.367896, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02939129195874557, + "trace_id": "d3891f42e67cd77c7c0566ca987bb718", + "request_id": "621110cb-6a44-45e8-91d3-26a71e0e0408" + }, + { + "utc": 1795102129.864973, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.006590707984287292, + "trace_id": "612888eb85220adebd54f45a4b920d75", + "request_id": "409d117a-a03b-43a2-b4b1-359d572bf27d" + }, + { + "utc": 1795102130.3706791, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.014915874984581023, + "trace_id": "afa57c5af16b279884430dbc1f7bd317", + "request_id": "93bccd2d-429a-420a-a6e0-86048da3feb1" + }, + { + "utc": 1795102130.864007, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.007295791991055012, + "trace_id": "8a2cf1d92931995308fcc9a6ebea6156", + "request_id": "9e8bc32c-1c0e-48f4-a7e1-e7e81066642b" + }, + { + "utc": 1795102131.372106, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01604704203782603, + "trace_id": "dc12395b6c9e8ef950641823c0d365f9", + "request_id": "f85bdd6a-4750-41e8-965e-ad7962e01708" + }, + { + "utc": 1795102131.372137, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02715741703286767, + "trace_id": "678fb2f6786e2c860031d5cec7d8d7b8", + "request_id": "f0ee10d8-4a8b-4b06-af09-d64f78e423c8" + }, + { + "utc": 1795102131.8736699, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.011406583013013005, + "trace_id": "270969ba3f19e45bb81ca91662d72d6a", + "request_id": "2beb4a25-62b7-4946-a757-f72c87af7956" + }, + { + "utc": 1795102132.370668, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01880133297527209, + "trace_id": "dd5277dbf4d2cfa4b27c70494fe34698", + "request_id": "8733aaa4-35e2-40f4-b120-4b3ab2e509ed" + }, + { + "utc": 1795102132.864602, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.006648999988101423, + "trace_id": "ef2ad9fc1cbfb819bde1292de21bbfc7", + "request_id": "d1c2f017-6ae0-4c18-8f5e-4a473d83e881" + }, + { + "utc": 1795102133.369617, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.020300833974033594, + "trace_id": "d8cc404ef6058310e200d26dd82b657a", + "request_id": "e8073a7c-512e-40b1-88ae-58c302345cb9" + }, + { + "utc": 1795102133.369664, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029653415957000107, + "trace_id": "86fdbcfccaae6dabce48452a43577ebf", + "request_id": "6c689cdc-2288-4e19-9fa4-8dccb54cd08b" + }, + { + "utc": 1795102133.873666, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.006441582983825356, + "trace_id": "48dff2908ef5fa420677bb77a612de87", + "request_id": "524fb113-b0df-4ad8-af10-d5be8f359859" + }, + { + "utc": 1795102134.373369, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.017398957978002727, + "trace_id": "128d1099f0de9ef678cf460798ef8a7c", + "request_id": "79870916-cb5c-4db5-b0bd-69bc1049b7a4" + }, + { + "utc": 1795102134.8704672, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01687654195120558, + "trace_id": "c88a4fb7153fcff3c4a4b63f2ff4c664", + "request_id": "08deafa0-d07b-4dd9-bdc9-78ca7fb6e0f3" + }, + { + "utc": 1795102135.3736682, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.018639500020071864, + "trace_id": "989f7e355720bd222c7c24d59ba28f20", + "request_id": "9f994de6-53e2-4a87-94a4-9e84c6e2954b" + }, + { + "utc": 1795102135.373708, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029731707996688783, + "trace_id": "f2dd95c2f354ba713ad3117d43b0bd73", + "request_id": "6d136112-469f-4514-ba7c-01b7197dcdd0" + }, + { + "utc": 1795102135.871721, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.005670249985996634, + "trace_id": "74ee5d0a5fd9a7441c577226f3df474c", + "request_id": "e4f4deae-9d10-459b-bf18-8398ad54abad" + }, + { + "utc": 1795102136.3711228, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.016543458041269332, + "trace_id": "4a9c1487ef2a4b506c295ee74e9ae7b2", + "request_id": "8cdf6c27-0adc-484c-b997-48b931cfd54d" + }, + { + "utc": 1795102136.8640158, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.015069874993059784, + "trace_id": "797b734bf683263c5f21b601aafdfe99", + "request_id": "5aec5c85-747b-4cdd-a0ad-17b76f66fff8" + }, + { + "utc": 1795102137.3658829, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.006932791962753981, + "trace_id": "230538cbb2036030b409fde49576a175", + "request_id": "6f30f1b8-7892-4e11-8356-a0c69a552429" + }, + { + "utc": 1795102137.365929, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011497250001411885, + "trace_id": "7461c53cff39d7a2c6e0ae8c15768770", + "request_id": "4a9ac853-bd9b-4e65-9667-bd07f27cc52c" + }, + { + "utc": 1795102137.871638, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.006090042006690055, + "trace_id": "19e1842c23ee4480205be11cc8ba1b4b", + "request_id": "98cbbc27-bf66-45f1-8096-59c8516c93fe" + }, + { + "utc": 1795102138.371719, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01631479198113084, + "trace_id": "30e80f1ee15266c680a9ffd5efd44b58", + "request_id": "ae3a55ff-bd89-46e7-858b-2df2b66ce158" + }, + { + "utc": 1795102138.871704, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.015870458038989455, + "trace_id": "d183b9a1ab7ab349a303e0242e4d1ae8", + "request_id": "9a9c2137-6d86-4469-9435-af115df0fcd0" + }, + { + "utc": 1795102139.37126, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.00871737499255687, + "trace_id": "aeeed9e9abfe952f83af3fb8b0bc16fc", + "request_id": "d8b6c5e5-e205-4028-ab79-6430cd56ccb9" + }, + { + "utc": 1795102139.3712819, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.014237333030905575, + "trace_id": "baeef8539829163a30d13fb950b13aac", + "request_id": "1dedd7ee-602c-4b8c-ac44-218f956dbb2f" + }, + { + "utc": 1795102139.872888, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.005246000015176833, + "trace_id": "681456940eb8644e8f5afae05749312b", + "request_id": "56be00d6-3130-458d-adc4-b553e687b1c5" + }, + { + "utc": 1795102140.373631, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.016968458017800003, + "trace_id": "e59207cf481a2a725c072b75c487bfd3", + "request_id": "4507c79b-2943-421e-a193-cef8441e283e" + }, + { + "utc": 1795102140.87009, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.016135042009409517, + "trace_id": "ed2331f8d5dec64d1d7962a8911d59ae", + "request_id": "9ce22642-6c74-4228-af90-e0abbda61287" + }, + { + "utc": 1795102141.368444, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.017838875006418675, + "trace_id": "bd378f273e48c2e95128f2f7dc484037", + "request_id": "ce86554a-ee46-4e23-a95c-c8f617489431" + }, + { + "utc": 1795102141.368489, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029024624964222312, + "trace_id": "8dc4107e7f5a8e4765d6fe80b003edc2", + "request_id": "cdd767a0-8e9b-4bf6-a113-237daab13db0" + }, + { + "utc": 1795102141.8702872, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.006725957966409624, + "trace_id": "8f6beec611b90f302a4aa08f0bebfca9", + "request_id": "0fedf0ea-92be-400e-8b2f-0d8f0bc1c8f6" + }, + { + "utc": 1795102142.367053, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.015737625013571233, + "trace_id": "139d1cb6312f211056ecaf4fbd239a11", + "request_id": "a6cc2ba8-241a-4bb9-94c4-395a3fc95059" + }, + { + "utc": 1795102142.869674, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.016150749986991286, + "trace_id": "d78dc54d5ab0d0cd2a9bd124168f6029", + "request_id": "3354217b-26c9-494f-8906-c0b4b63d3673" + }, + { + "utc": 1795102143.370038, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.02062245801789686, + "trace_id": "6d6fcc3cf45fbeaf0b29c6fd2b239cfa", + "request_id": "9ff1f2de-0a1d-4254-a2c1-ba871d71c603" + }, + { + "utc": 1795102143.3700821, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.031239625008311123, + "trace_id": "56c4216d69bca4cccad6a04990a262d4", + "request_id": "2a8d79db-1355-43cf-871f-c33d83728af8" + }, + { + "utc": 1795102143.870271, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.005630084022413939, + "trace_id": "4bbc403a963294ca4ca8ac8c297386bb", + "request_id": "149273ba-e626-4ade-b2b7-6ddeec5f1848" + }, + { + "utc": 1795102144.3716722, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.018851166998501867, + "trace_id": "265828e60f8d304860d7e59d497b6ca0", + "request_id": "b76ecd82-17ed-4563-bc79-4155301ed699" + }, + { + "utc": 1795102144.871309, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.016887207981199026, + "trace_id": "a954dffcafc8614382acc18bb0ac9ac8", + "request_id": "268b215b-c318-4e69-9413-a55610207249" + }, + { + "utc": 1795102145.371719, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.017538332962431014, + "trace_id": "873899f96073ade473e4e60457c844b0", + "request_id": "cd2b9474-ac96-4287-89de-399f15a0b1dc" + }, + { + "utc": 1795102145.371765, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02869141596602276, + "trace_id": "ff3de56d0ee386135f759910d838b25e", + "request_id": "7685bc3e-fc5e-4b92-ac4b-6470a238bba6" + }, + { + "utc": 1795102145.871593, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.007074250024743378, + "trace_id": "06f19b0054da0b0cb86e621d1f3a1044", + "request_id": "f1b7c820-999d-4f72-9f5e-ada114bf14a6" + }, + { + "utc": 1795102146.3645408, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.015526041970588267, + "trace_id": "de40d86403b589f111aae7d332c3d1bf", + "request_id": "88f4e9bc-f13e-4418-bd14-ac4ef416b41b" + }, + { + "utc": 1795102146.8665211, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01678262499626726, + "trace_id": "7211ef05d4d3818ff7e1f5ec1a50e1c4", + "request_id": "588cb4c7-cf29-411f-b205-10e7fc9e75aa" + }, + { + "utc": 1795102147.3716888, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01767845801077783, + "trace_id": "42e591899636a40de42f5a2ae6c83732", + "request_id": "13d507fb-08bd-4957-a8d0-5c2b3df462da" + }, + { + "utc": 1795102147.371734, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02974045800510794, + "trace_id": "cc3a830a68914c126443dceaa2b7d817", + "request_id": "2755c99c-211a-494c-9f73-21b96a21601a" + }, + { + "utc": 1795102147.872915, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.006056250014808029, + "trace_id": "c36777d52f4d23f4ca19db48ddaf7b35", + "request_id": "aa68992a-00db-46a5-a15c-f088b3d1501d" + }, + { + "utc": 1795102148.371679, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01698050001868978, + "trace_id": "e0ed5b0e4a43bc0be6249e35f985b8b8", + "request_id": "ffb721c6-1287-4c8e-a724-af28b9aafdd5" + }, + { + "utc": 1795102148.868237, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.015908250003121793, + "trace_id": "7b06ecc9f4b629cab9ef188be518dc45", + "request_id": "a6adc910-d5f1-46ff-bd37-e31b1a5dd537" + }, + { + "utc": 1795102149.373627, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.019483999989461154, + "trace_id": "d06350c8bec68a97c9f433a8bf459e9e", + "request_id": "4a38605b-67e6-45cc-a9b3-2409e4ad2cec" + }, + { + "utc": 1795102149.3736699, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02978120796615258, + "trace_id": "3c1a531eabc1ad0ede1a3571c4300e7d", + "request_id": "3a7a9000-4073-4372-aa81-d206e7089789" + }, + { + "utc": 1795102149.864116, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.013354707974940538, + "trace_id": "66a033286c7eee48cf3b4d91fd5ebe6b", + "request_id": "329337eb-7d7c-4b07-9d9d-0074244fa11f" + }, + { + "utc": 1795102150.373659, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.011396625021006912, + "trace_id": "eec1d2811731fde1f0ca67ab2c5319b0", + "request_id": "2027ff10-32d7-44eb-806d-ef7cc78db7c8" + }, + { + "utc": 1795102150.871653, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.01634595904033631, + "trace_id": "817730fb5e200605db259a72f69b9b28", + "request_id": "9a37a390-b68f-4ffd-8d71-2128f1a8d0a9" + }, + { + "utc": 1795102151.369639, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.0172840419691056, + "trace_id": "1050b2a24a01fb038dd0ab1e0cf52b1b", + "request_id": "30acfc28-c1ae-4509-ad8e-99d37ca98cc1" + }, + { + "utc": 1795102151.369683, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030930459033697844, + "trace_id": "abb7d934dda8d2172609a7db86f71f86", + "request_id": "232b08b4-0e45-4c46-955e-c46c765c2053" + }, + { + "utc": 1795102151.873515, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.0071858749724924564, + "trace_id": "3212198c923909c246ca398702d26389", + "request_id": "e87a4e3f-d2bc-44ff-9b98-af03dbffcab2" + }, + { + "utc": 1795102152.367168, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.0055832500220276415, + "trace_id": "d4ae3e732946773f6d936eae777534d8", + "request_id": "a7028519-4b76-46b5-93d1-1cd9410da6d2" + }, + { + "utc": 1795102152.87164, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.015000040992163122, + "trace_id": "f5bc10ce5396dc3340d0e2861a971735", + "request_id": "8746ccae-5960-492c-bc19-090b7f501cf9" + }, + { + "utc": 1795102153.369698, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.012320458015892655, + "trace_id": "3beb10298116e9e083ed98b00dcf3f9e", + "request_id": "0fc84921-3fae-4208-ae37-a734d7376978" + }, + { + "utc": 1795102153.369729, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.025170708016958088, + "trace_id": "8d6ec8250b30126d79c978ad962c24fd", + "request_id": "e2fdb294-19eb-4253-b7ea-204816c9880e" + }, + { + "utc": 1795102153.864363, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.005915875022765249, + "trace_id": "39e03b94978826ba32ba07192f8c9ba5", + "request_id": "ff1c2823-8425-42c6-833c-42e821839c95" + }, + { + "utc": 1795102154.370425, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.011507250019349158, + "trace_id": "b03a0d7ff4b7f0fbb48d7b71b56ae855", + "request_id": "8407401e-9edb-4efc-a6db-b9cf4c816fe0" + }, + { + "utc": 1795102154.8658009, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.015921375015750527, + "trace_id": "2fadf69c978156d703b9b59902da69ac", + "request_id": "dcc417c3-fe99-4c21-8fcd-91ba58722082" + }, + { + "utc": 1795102155.370494, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.018147709022741765, + "trace_id": "9e3d5937c2ce582b9c1a94abd24cd5a0", + "request_id": "0d6b1800-5301-43c4-a39c-c91c55a85229" + }, + { + "utc": 1795102155.370537, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030008375004399568, + "trace_id": "a334b5da72e18ffa25e0e641d79016d4", + "request_id": "4b80498b-1944-48b3-ab82-014cf5936c7e" + }, + { + "utc": 1795102155.8663812, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.004871833021752536, + "trace_id": "405212f5fc42ba23948b1c729efed790", + "request_id": "67f25289-9680-49b0-bad0-ab3bea7b91cb" + }, + { + "utc": 1795102156.371147, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.009523166052531451, + "trace_id": "eb4434e4fb85d80d6c83cdbe60262aee", + "request_id": "333c1459-ed77-43fd-9032-9440df6dbe94" + }, + { + "utc": 1795102156.873591, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.014187499997206032, + "trace_id": "582dc6fcf92914c959e6a7495fae6f90", + "request_id": "a4cc7d37-0e71-4951-8f2d-ac32b0e438d6" + }, + { + "utc": 1795102157.373607, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.019571749959141016, + "trace_id": "48d6ca8d4b0e6a76f2f582a4b68162fb", + "request_id": "4c915a5e-0dcc-41da-9533-5f1ecd8ca634" + }, + { + "utc": 1795102157.3736498, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.031605291995219886, + "trace_id": "a3fd44c0e4d34843fad0c741fe697a53", + "request_id": "0f33e6d9-74ab-43b3-9aee-701a7a944d23" + }, + { + "utc": 1795102157.871058, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.005485708999913186, + "trace_id": "eeddd6738f52475bb3d4fe0941434be4", + "request_id": "4bd7768c-01c9-49cb-9ab9-04d57e10ba6d" + }, + { + "utc": 1795102158.36674, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.016775708994828165, + "trace_id": "6df4945c87a5b2c83fbd817b93adc81c", + "request_id": "97eb295f-41cc-4d6d-ac61-f7169dea32ac" + }, + { + "utc": 1795102158.87042, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.006331208976916969, + "trace_id": "469da865572375eb1180b9f051be0ba2", + "request_id": "7d491387-b39a-4cbd-be1c-35334d022f89" + }, + { + "utc": 1795102159.3704388, + "phase": "after_short_spike", + "kind": "message", + "status": 201, + "duration_seconds": 0.02101412496995181, + "trace_id": "d3c7bbc9c0ff07ea4b2ab7399b14287f", + "request_id": "9ca1018e-3473-455f-acdc-cb253f2e0d14" + }, + { + "utc": 1795102159.3704672, + "phase": "after_short_spike", + "kind": "upload", + "status": 202, + "duration_seconds": 0.032762582995928824, + "trace_id": "679ecfbcd899d9ed26267a0c500f9b92", + "request_id": "f70b268a-74cf-4336-b59e-734629b08347" + }, + { + "utc": 1795102161.370847, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03127991600194946, + "trace_id": "17b7581bbeff341c23184f5121066d09", + "request_id": "3246fabf-531d-42e5-be46-a3dc445e133f" + }, + { + "utc": 1795102159.871603, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.005742833018303, + "trace_id": "d136f5839104de300e16b8914cc0c1bd", + "request_id": "3b091e8d-962d-46ca-b3d1-ceb44f4478b0" + }, + { + "utc": 1795102160.3695538, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0149917919770814, + "trace_id": "2aad1d15e656c72f163bf42c6ed6650a", + "request_id": "a2ef1171-8b26-4564-b895-00e6c48ba773" + }, + { + "utc": 1795102160.871642, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0185394170111977, + "trace_id": "c38b1108bd4bb46777cfe68f620d6bb0", + "request_id": "7fc0bd15-0287-495f-9fa4-d272da0fcef9" + }, + { + "utc": 1795102161.37057, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.016247417021077, + "trace_id": "75f788013ee9b9fc36a361fcbcbe7992", + "request_id": "21420f29-ff6c-4e6a-9c16-3fe00dcfec40" + }, + { + "utc": 1795102163.370173, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02778904204024002, + "trace_id": "346a72f93fa471dfb1fc9cf8b9d09074", + "request_id": "843b5088-6151-4104-850f-6fd362291b68" + }, + { + "utc": 1795102161.8735092, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0074133339803666, + "trace_id": "1f4bbd308526edf57bdbc9924c173ea4", + "request_id": "2247b9b3-7217-4a38-a1cd-6e0fbf7730f7" + }, + { + "utc": 1795102162.369596, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.019151917018462, + "trace_id": "0ce649a8e04c306386bf16b70042e3bb", + "request_id": "8be2c2e3-1d81-4a45-aa04-df87b6084b44" + }, + { + "utc": 1795102162.870202, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.018525708990637, + "trace_id": "bf2d07d805a39f94e16d86d641928c9e", + "request_id": "7939f43b-15f2-40aa-87d7-a5b647bac39f" + }, + { + "utc": 1795102163.3698099, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.013112209038809, + "trace_id": "212994360c631a0728e120c631b1b399", + "request_id": "e09ea534-57bb-4731-93ea-509f9adf2791" + }, + { + "utc": 1795102165.371572, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.015355707961134613, + "trace_id": "d7f8b9e6a8fed5c50f3f64dba4c93d61", + "request_id": "85341e70-7902-4877-bbf9-d6ed414bb673" + }, + { + "utc": 1795102163.8712692, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.009232249984052, + "trace_id": "d1b30c14c65d29b3f7949b821de2a577", + "request_id": "457e0b5b-1b39-45e7-aa26-7eae3978d106" + }, + { + "utc": 1795102164.369604, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.016777375014499, + "trace_id": "5ad7d9550fa0938ea04d8616f6ab41ff", + "request_id": "e8e9ffcb-9ff3-4ce1-9805-b2b00faaa109" + }, + { + "utc": 1795102164.872583, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0220262500224635, + "trace_id": "1aefc6b8e2af6e088febf782fa8b2dfd", + "request_id": "b21c897a-b5e3-4f77-9edd-6ebe5f6e17a4" + }, + { + "utc": 1795102165.371537, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0129287079907954, + "trace_id": "2ef956960549727daf08d8cb82b38a16", + "request_id": "46cc40cc-0973-4802-9d8c-b41675620c64" + }, + { + "utc": 1795102167.365702, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.035390666977036744, + "trace_id": "39391ec85cd9df79c66d3cfc07cb70a8", + "request_id": "f69b9fac-b337-4287-9ab0-afde204b8ba7" + }, + { + "utc": 1795102165.871556, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.006952708004974, + "trace_id": "7224d6b134c5fc2bfb748835f5cb61a3", + "request_id": "6b98a687-a706-44d5-ac15-c9217f0e1160" + }, + { + "utc": 1795102166.370332, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.015763917006552, + "trace_id": "480edd361ab0f2c60c6cd013d44d6508", + "request_id": "50b5be3b-83a7-48ea-b5ed-7fc5d79625c5" + }, + { + "utc": 1795102166.873558, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.01695895800367, + "trace_id": "2576cf5f5d5718ffbb94bf6fa52224a9", + "request_id": "0edb55ae-1f3a-46da-9fe6-bf67c9140917" + }, + { + "utc": 1795102167.365673, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0220558749861084, + "trace_id": "b3f55227b63777c0d4af48e77f2aae24", + "request_id": "710d1aa6-fd96-4aeb-a360-59882e37696d" + }, + { + "utc": 1795102169.373585, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03243937500519678, + "trace_id": "ca2b778e3e7beb1fb1b038b59d6fca1c", + "request_id": "c50134a7-008c-47e4-a344-e1ec9163efd0" + }, + { + "utc": 1795102167.86604, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0080604160320945, + "trace_id": "3e30029f7eddbbc27baf63f0187a2025", + "request_id": "c17fbeaf-864b-4137-ab08-9f6f68d8539c" + }, + { + "utc": 1795102168.371606, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0181651250459254, + "trace_id": "ee589505a98cf3358fe673f12b83a8b3", + "request_id": "f2568df0-b9f9-4c81-bbfd-dcc0295264a4" + }, + { + "utc": 1795102168.871603, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0161381250363775, + "trace_id": "8502e16cd7af21da57234636efdb344b", + "request_id": "1975a1a7-6155-4c86-89af-f4c0ad38a015" + }, + { + "utc": 1795102169.373546, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0169495420414023, + "trace_id": "2a4d16cdfb82a670df5a3776bf0e5844", + "request_id": "bae97bc2-0217-4c9c-a77e-79c7644c08e9" + }, + { + "utc": 1795102171.3659542, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03287104197079316, + "trace_id": "e9544119e90eb74fa3c7237e694575f3", + "request_id": "7cc09c47-cc94-4051-9035-5c17ad42146d" + }, + { + "utc": 1795102169.871542, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.006110125046689, + "trace_id": "6c6356f303985741cd6eefca79038cec", + "request_id": "c40a5171-04b8-4cda-bb55-219f5085cd44" + }, + { + "utc": 1795102170.365727, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.009495791979134, + "trace_id": "3c1f06f945f94e696a79c517f06c7408", + "request_id": "0fe00102-3605-43cc-9ef1-b259719e859c" + }, + { + "utc": 1795102170.8647392, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0177297919872217, + "trace_id": "9a1c6ec74794127b0b790984235193a4", + "request_id": "794a2c8d-b9c9-4ed7-9a4a-3a0a914deb80" + }, + { + "utc": 1795102171.365914, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.019947749970015, + "trace_id": "57862b33e9e20bfedc3ac7632562adfb", + "request_id": "13fda9eb-da7f-4e23-84ec-d924ddf3d8c8" + }, + { + "utc": 1795102173.371688, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03351191699039191, + "trace_id": "d788af8c8aa0f80c893fbb3e891401f1", + "request_id": "3fd34fe3-19ab-45a5-ba13-8069c5f2eada" + }, + { + "utc": 1795102171.869535, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0072351670241915, + "trace_id": "5d9bcb1964ad381ea07afa86af43bf0e", + "request_id": "fd3d0489-51ba-4129-a462-18f9f7160150" + }, + { + "utc": 1795102172.371569, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0088811250170693, + "trace_id": "8a994f96653dbad78e72b1a93186efcf", + "request_id": "d4ac1da3-bb4d-44f4-a009-7f4a9e131571" + }, + { + "utc": 1795102172.869519, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0200541670201346, + "trace_id": "4755519754e145310e9cfe4eb3fdabc7", + "request_id": "261b7b16-eb63-4a09-97ba-74d543e427f5" + }, + { + "utc": 1795102173.371646, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0177328329882585, + "trace_id": "6c2a0904316c8600c88098d37551aaf0", + "request_id": "0febfb7f-e352-41f0-b992-c6648f9527b3" + }, + { + "utc": 1795102175.371607, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03443949995562434, + "trace_id": "0e54ce57b07ab6b264c7dfe4658d48e0", + "request_id": "530a81d0-bb8a-433e-9ccb-1bbdf4ee326a" + }, + { + "utc": 1795102173.871477, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0084170419722795, + "trace_id": "91dbfcecb9ab1465aceed6b34b15c615", + "request_id": "60ed5c79-9ad0-439f-b1d1-3ec6a77c8c55" + }, + { + "utc": 1795102174.3701031, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0212019159807824, + "trace_id": "cf6b300eace29b1ad7e4e536d2492de8", + "request_id": "75035dfc-51ce-43f5-bd08-637a44ac148f" + }, + { + "utc": 1795102174.871549, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0159510420053266, + "trace_id": "f61c17c1e835fb9b712720810a71523b", + "request_id": "482e6bb5-df5b-4733-9a4f-035356a7b311" + }, + { + "utc": 1795102175.371566, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0209830829990096, + "trace_id": "8c4282425ca2340a1a1a2238cfa8a3a4", + "request_id": "4cf3cd58-db1d-490d-915c-29c9279cb1fc" + }, + { + "utc": 1795102177.366032, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03378412500023842, + "trace_id": "96726f58f12c843a28db405fd9dccb10", + "request_id": "4b131314-abb1-4bc4-a237-d36ec8ae41f1" + }, + { + "utc": 1795102175.8667, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.006491750013083, + "trace_id": "ca7b1d229547f79c4cebf81e35f821f3", + "request_id": "4b35f9f4-419f-47e6-aa54-ad350ea90dd7" + }, + { + "utc": 1795102176.3695612, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0315085420152172, + "trace_id": "31aab5d58311f9dfa50431d8fb2f50c0", + "request_id": "661e9fa6-59e1-463b-83bd-7dce170440ad" + }, + { + "utc": 1795102176.871553, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0212873749551363, + "trace_id": "d3e19166ff43d687ed7698693d09bac9", + "request_id": "d968778d-9aae-4b67-b67c-bbd9f23416ab" + }, + { + "utc": 1795102177.365993, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.022267959022429, + "trace_id": "e26dffee65e2edb87d70c248bc92e994", + "request_id": "a306ffb1-15dd-42c6-84f7-3506fffc8f86" + }, + { + "utc": 1795102179.371595, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03428975003771484, + "trace_id": "ac0b31417d24cbaf7ac39e1e81e9dff4", + "request_id": "6f39ab30-1901-4700-81f6-ce8148494117" + }, + { + "utc": 1795102177.873353, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.005880791984964, + "trace_id": "54b8c48fcf5c5fa20b5744ed9e70b9c9", + "request_id": "1d551cf2-abd9-419f-acca-3cb261f58a73" + }, + { + "utc": 1795102178.371548, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.007307125022635, + "trace_id": "b3f21d706fd53ffcdac6d865d8ff28bb", + "request_id": "cb792e02-7370-4624-acae-af585dba1f17" + }, + { + "utc": 1795102178.865582, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0195727910031565, + "trace_id": "33ee784e857853d6fcf5cf9ed667362a", + "request_id": "520e5141-a0fe-4c33-8954-b2a7ab001b05" + }, + { + "utc": 1795102179.371556, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.02095991600072, + "trace_id": "d0d08d33cc5b0059f8c93996404210a7", + "request_id": "c1cd666d-f92d-4b8c-b8d2-7b0756b85255" + }, + { + "utc": 1795102181.366105, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.0324831249890849, + "trace_id": "f6596c63920555dd043ee1437b6b68fc", + "request_id": "5dbb27e2-5875-4f89-875a-07472ca772a8" + }, + { + "utc": 1795102179.8734279, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0093378750025295, + "trace_id": "74dd0c675ba91599cbcbd275d6cfc839", + "request_id": "f7536aff-23a9-4a24-8560-3b7568a51d2e" + }, + { + "utc": 1795102180.3734112, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.010635041981004, + "trace_id": "b15afb2a9a3c75a6c78d33235ef374e7", + "request_id": "ca650e9b-3032-447e-9556-cc4c9be23fe9" + }, + { + "utc": 1795102180.869473, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.023254416999407, + "trace_id": "c7f2661a0b95e4d06f9973061bede252", + "request_id": "d1f620a5-9d1e-4ac3-b676-be916e41781a" + }, + { + "utc": 1795102181.366063, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0170874160248786, + "trace_id": "e1ad217989481436284fb64a3934eb5a", + "request_id": "b5f01902-754f-447b-a0ce-c097f079f5eb" + }, + { + "utc": 1795102183.363779, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.033670791948679835, + "trace_id": "f85912ec0d10336e480c097822b080ec", + "request_id": "d256a6c6-f79f-49cb-871f-486c87cbea51" + }, + { + "utc": 1795102181.869743, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0087325000204146, + "trace_id": "2dce15805365ab8acf50cc98ec2270dd", + "request_id": "8e9c50c1-6be2-4762-9ec2-53252a74bdb7" + }, + { + "utc": 1795102182.3715441, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.020128707983531, + "trace_id": "8aee33dedd62b7f850ae668e75328798", + "request_id": "fba5fc8d-e176-44de-a678-2b0266a3ce76" + }, + { + "utc": 1795102182.867156, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0173964999848977, + "trace_id": "af71ebc794d32c0c7d03385cd912e164", + "request_id": "3fab8cff-825b-4c56-9ec1-049c0cd85d90" + }, + { + "utc": 1795102183.363743, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0204468749579974, + "trace_id": "f585e8739a3f0438dde3ccb4e1aeeb94", + "request_id": "225af768-24e8-4a44-970b-58a8fee3de8e" + }, + { + "utc": 1795102185.371575, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03398875001585111, + "trace_id": "75cd9245e787b9134e9b9de9397ef578", + "request_id": "dd9c3d19-6e90-4bb7-9f7d-895e31a200b1" + }, + { + "utc": 1795102183.8715498, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.00872362498194, + "trace_id": "02c2167a1a6098e60615c9689d5c19e4", + "request_id": "0d817d07-3dc4-4212-8f1e-fc1a6cc159ea" + }, + { + "utc": 1795102184.366195, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.017403709003702, + "trace_id": "a94d9bb35925c642393326a8ac911367", + "request_id": "b30d3f7f-2b25-499e-a53f-c228cf7410a1" + }, + { + "utc": 1795102184.8735092, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.010488875035662, + "trace_id": "e38d8725c79045763fea80f0e06645af", + "request_id": "fbd6e696-4fee-455d-9e7b-d6b80ba697a2" + }, + { + "utc": 1795102185.3715358, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.02215920900926, + "trace_id": "291e4c722d455e3b1e1344658900d05c", + "request_id": "09aa3593-bb13-4efe-a02c-98d6b98ffcb0" + }, + { + "utc": 1795102187.364327, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03414091595914215, + "trace_id": "094f6de2a399ed1b6faf537adcb2a5c8", + "request_id": "0d1b17be-fafb-45d0-a191-6b8e312e5907" + }, + { + "utc": 1795102185.8715441, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0068041660124436, + "trace_id": "0a0131385ba59d55025403d6f770c33e", + "request_id": "88e7133e-2878-4189-97e7-f17e9402804b" + }, + { + "utc": 1795102186.370124, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.022254500014242, + "trace_id": "2b1b4377d081f264dbf772af4afab5e6", + "request_id": "e8ddc3c7-62f0-48c4-addc-37460b649301" + }, + { + "utc": 1795102186.865587, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0130209579947405, + "trace_id": "f17aecae27da12c4b6493e7e666c9ac8", + "request_id": "5660c821-350c-4707-8745-5e8bed4d3c02" + }, + { + "utc": 1795102187.364286, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.021992499998305, + "trace_id": "d2d7c1327a791652f942011d5f20836e", + "request_id": "dcdec65c-6ff6-4b70-8ddf-a80778bb2870" + }, + { + "utc": 1795102189.37159, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03357362502720207, + "trace_id": "547ba5b30a15d774befe032863eea053", + "request_id": "01c414dc-6ddc-49c9-b21f-23a2bf0cca72" + }, + { + "utc": 1795102187.8714292, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0061534579726867, + "trace_id": "de2b80f6f8a15663b2d95414d6f7d97d", + "request_id": "de5daa84-9f3b-4041-869d-044a6958fb28" + }, + { + "utc": 1795102188.371467, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.018173791002482, + "trace_id": "b2d3b95a52232957ba7270673a1d1a6c", + "request_id": "c36f3b60-ba9b-4d50-a54f-4cbb8ec2a779" + }, + { + "utc": 1795102188.8694432, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.021584750036709, + "trace_id": "22ec6333f73a66044be7fca2253b7da4", + "request_id": "807f31ba-7274-47dd-82c6-3114b60686d1" + }, + { + "utc": 1795102189.371548, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0214595410507172, + "trace_id": "3a2169b434b2978eecdc2a4534a87a10", + "request_id": "075519da-e2dd-411a-b660-f80afcdfec3d" + }, + { + "utc": 1795102191.371256, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02331137500004843, + "trace_id": "73a5a23a83dc1b58e2076f80e4ae41fe", + "request_id": "8884e688-584a-4d1e-baa1-0271bab237a4" + }, + { + "utc": 1795102189.8714151, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.007366749981884, + "trace_id": "5afd799f93bb690988cda4a4a4c09736", + "request_id": "e87e8a1b-60c6-4178-b7ae-d04b88e5f786" + }, + { + "utc": 1795102190.3660128, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0184045410132967, + "trace_id": "e24168e5251266f8535ad8db7fbf8d38", + "request_id": "669d4735-6ddd-4b81-8d99-1c625a1fe886" + }, + { + "utc": 1795102190.871451, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0178856670390815, + "trace_id": "447bd8090bef44af45ecf80ba000a386", + "request_id": "9c6b8a55-3ed6-4b04-9577-6e0681354d45" + }, + { + "utc": 1795102193.364065, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01561674999538809, + "trace_id": "e2427439daafeff68bcae08411edebec", + "request_id": "aef734c5-b15c-4bb7-9b56-0914b3fe5045" + }, + { + "utc": 1795102191.371223, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0095641249790788, + "trace_id": "83cf9682686b8dcb7532d4965cef3050", + "request_id": "ed049d3a-e4fe-4258-8ec9-8af36e438bff" + }, + { + "utc": 1795102191.865471, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0108155829948373, + "trace_id": "3c76d2145277cc8b9953e52e49a57828", + "request_id": "f144d4fd-3d85-4354-a319-19f29198426d" + }, + { + "utc": 1795102192.373492, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.020265124971047, + "trace_id": "c28bc1c00ee2931f38c2ad637282b73d", + "request_id": "cbbad1d0-217b-42ff-8df3-d066890ea303" + }, + { + "utc": 1795102192.873468, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.019503458053805, + "trace_id": "0b574018961d2e8ab4e43b688166a26b", + "request_id": "786b8468-79e0-4703-9709-39e20c378af4" + }, + { + "utc": 1795102193.364049, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0107094579725526, + "trace_id": "f30f41e0e50092cd66143791c2827292", + "request_id": "257d0d4e-6a1d-4a9d-9505-2516b2d3fd9f" + }, + { + "utc": 1795102195.369719, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03577541699633002, + "trace_id": "6a193b85df134e8e0c201fde1310fb36", + "request_id": "cecc43ee-edac-42d5-8b05-9ff8a0235cc6" + }, + { + "utc": 1795102193.8714979, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0101971669937484, + "trace_id": "a31e9a2c0952dfe448cab6b80811101d", + "request_id": "b96ff15a-e98b-4a5d-aad3-bda579445e35" + }, + { + "utc": 1795102194.369403, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0172163330134936, + "trace_id": "5d387e329290c06f873fe6ca067bf5de", + "request_id": "cfd901cb-a314-4af1-b7af-431f0cdbe30c" + }, + { + "utc": 1795102194.865637, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.00961616699351, + "trace_id": "fdb2bece4f90223288f96a33d3f76fee", + "request_id": "b5f73bcc-743d-4ad3-bcd8-f60df81f7a5b" + }, + { + "utc": 1795102195.369674, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0299333339789882, + "trace_id": "43ae75e45aa6491845f01ff9a55e62be", + "request_id": "28c3abc0-e04f-43dd-bc7f-30a4ec8b6f78" + }, + { + "utc": 1795102197.369518, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.036773167026694864, + "trace_id": "cc13603006a9cbada3aed6f2a15c6d86", + "request_id": "5f503f1f-d9b9-431d-b26e-c2df2cfc6122" + }, + { + "utc": 1795102195.873417, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0085795409977436, + "trace_id": "314b00d70b1af5088984377c8a0652a0", + "request_id": "7baa59ca-4c42-4d68-829f-03f39c530dcf" + }, + { + "utc": 1795102196.369478, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.016824375023134, + "trace_id": "724bbdaba3cadcbf889bf78325a4e961", + "request_id": "fca421a7-5200-427c-990c-004edfbb2fef" + }, + { + "utc": 1795102196.8656461, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0099163749837317, + "trace_id": "063c557d9be028091b90f526e2104f96", + "request_id": "fc0a00ff-62f5-4f45-b0d6-c276f6874968" + }, + { + "utc": 1795102197.369476, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0216787079698406, + "trace_id": "493dcb2e7a401ff13d816a4818e8adbc", + "request_id": "4a9ba121-ede7-4c5e-8a7d-e47b9c243360" + }, + { + "utc": 1795102199.363544, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03149087500059977, + "trace_id": "2d029ad2ca2655dfdfd60d9f38de659c", + "request_id": "db5086c2-542a-4b61-94a4-f0412e213658" + }, + { + "utc": 1795102197.8713238, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0060998749686405, + "trace_id": "876c902d2bd8fa374750b1d717f9c574", + "request_id": "70476141-0f44-49fa-9669-63abed5762de" + }, + { + "utc": 1795102198.373444, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0220313749741763, + "trace_id": "5e1737d93e7847b2659597949c30c1ca", + "request_id": "8ee53e3d-d89c-443b-ad76-20488fac991b" + }, + { + "utc": 1795102198.8714619, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0106421670061536, + "trace_id": "35c773f6e8ca327868386c23b2c3ad56", + "request_id": "e5eb1e4b-5a80-492d-93f8-1b802933a1ac" + }, + { + "utc": 1795102199.363505, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0206372500397265, + "trace_id": "831178cc37e4085ddec224972e6b5f9e", + "request_id": "97415e28-2657-4c7b-aabc-5bdf85c49fc8" + }, + { + "utc": 1795102201.372719, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03482699999585748, + "trace_id": "271a18873bc14b84f379667c5f440934", + "request_id": "a4b32e0f-565c-4ff3-9f8e-e11ecef3b583" + }, + { + "utc": 1795102199.873368, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.007804416003637, + "trace_id": "212207a467e235453482fb9ad7e2d0bd", + "request_id": "270e77ad-b72f-4d3e-822e-de0c01cf83d5" + }, + { + "utc": 1795102200.370431, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0207848750287667, + "trace_id": "e6c730565e878bbe03429fa432280c0d", + "request_id": "4ce4b572-aefb-4a05-8852-fb186baa9fe0" + }, + { + "utc": 1795102200.864146, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0076445000013337, + "trace_id": "790287b8363c16204a8d5fa9bdf25f82", + "request_id": "04e821dd-ba4f-4ab4-a741-6befd8a0c12d" + }, + { + "utc": 1795102201.3726778, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.022544832958374, + "trace_id": "41796e12392badd384bd1fc4c5517e46", + "request_id": "8e135690-5b7c-4cf0-a254-82e8df0a8472" + }, + { + "utc": 1795102203.3694358, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03180491703096777, + "trace_id": "056ba2d3bc324a5cc43557649bd831b9", + "request_id": "07cfe729-c657-4a0e-8d7d-c1b390854e15" + }, + { + "utc": 1795102201.8646662, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0082286670221947, + "trace_id": "a29fac0c54ea4dc2d177541d7d09d38b", + "request_id": "21909187-6446-49a2-8bbd-c1f3d8d764c0" + }, + { + "utc": 1795102202.36559, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0148785420460626, + "trace_id": "1d6f23d2d288d9557288c8699f62540c", + "request_id": "f6d00b01-7de6-4a41-bb3e-b1d398c3e29a" + }, + { + "utc": 1795102202.8670888, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0229939579730853, + "trace_id": "e0edd55a2d2a58741cd2049e55f7429a", + "request_id": "f22a10f2-9817-4a71-aca3-b11040952ce4" + }, + { + "utc": 1795102203.369401, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.018068834033329, + "trace_id": "9c1bcac74020ed3c6a452d66f80125e0", + "request_id": "fe622306-4572-49a0-8956-613b4f943262" + }, + { + "utc": 1795102205.371487, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.06526645796839148, + "trace_id": "878018b6f548a9251a1dd697c90efd7f", + "request_id": "49699b28-615d-4971-8d8f-0c913343a103" + }, + { + "utc": 1795102203.864135, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0099628749885596, + "trace_id": "d333e047ff7a3c00de82611c11dab2d0", + "request_id": "fff8e48c-c150-4b04-99b6-36c8e25d6d3a" + }, + { + "utc": 1795102204.365466, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.01399612496607, + "trace_id": "dd087393b3d35475f1737a369a03e0b3", + "request_id": "5d4e35f1-7c06-4a26-a8c2-4a853ce46b94" + }, + { + "utc": 1795102204.873369, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.025757875002455, + "trace_id": "e5ac7d1ac9f98b0e95f8bd40ca5badd4", + "request_id": "5bffb5de-3fe7-4d47-b0ee-54e81fb011ca" + }, + { + "utc": 1795102205.371445, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.024821208033245, + "trace_id": "2c2de35d0ccfb73a8c49f0d039cadaec", + "request_id": "43a5cacb-5ea0-4fb8-b5e4-4041d4e425dc" + }, + { + "utc": 1795102207.3694878, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03414050000719726, + "trace_id": "8a68a779e21636e72d6afcfcfde2b5a6", + "request_id": "70324596-9a05-421a-9f79-4af71b34b700" + }, + { + "utc": 1795102205.871449, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0068042500060983, + "trace_id": "e43321f3defc4831467a92e0a1b0d86b", + "request_id": "746f4d84-1c22-4c8c-85c6-143a5693639b" + }, + { + "utc": 1795102206.3734798, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.007207666989416, + "trace_id": "60c91884a3a4854efb9781d11e47586c", + "request_id": "33c1bed6-34bc-4b3f-99b4-8c65ae13418f" + }, + { + "utc": 1795102206.8693051, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0167527919984423, + "trace_id": "c68aa258210f49de0ab393b041b65a69", + "request_id": "f0e27735-b35a-4d56-8100-665f501cf524" + }, + { + "utc": 1795102207.369449, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0178250419558026, + "trace_id": "d0797fe8cbac8b589e304b06829147c7", + "request_id": "a35d85a1-c913-4b3b-b45d-0e49d067c2f4" + }, + { + "utc": 1795102209.3693209, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029569708043709397, + "trace_id": "e2a5fcce6a8b6fd1cf4ebebc8e5ad8ba", + "request_id": "d5ecbcdd-7f34-4d12-9482-1ae22217bf48" + }, + { + "utc": 1795102207.871451, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0072151250205934, + "trace_id": "f306a5e51a6a3754d4ae3182a6e5cba9", + "request_id": "e9126055-6777-4cd8-87b7-26bb93104c11" + }, + { + "utc": 1795102208.369353, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.011778125015553, + "trace_id": "0d99afb747888c7664e4f2ae3946d849", + "request_id": "d13fb16c-523e-4f8d-89a6-baed51e3d40d" + }, + { + "utc": 1795102208.8658261, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0170574170188047, + "trace_id": "745749b7232ce63539b76989a3e6e7d0", + "request_id": "4a42da9c-1378-4025-9f97-5648d11246ae" + }, + { + "utc": 1795102209.369281, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0167667909990996, + "trace_id": "150d51d3a983219f3a3e0dfc2e684a6b", + "request_id": "199cbd03-9535-4d61-9d0f-469cbc54df36" + }, + { + "utc": 1795102211.370063, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03652849997160956, + "trace_id": "46f1013be7e49e589ffbf5580e95e667", + "request_id": "f6f57245-9c86-4d9b-b029-097075e8df1a" + }, + { + "utc": 1795102209.871347, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.006230792030692, + "trace_id": "1e8637146e0420aa36df2bc1000d2b7f", + "request_id": "2b70af46-b518-42e6-8e57-bf20522fd8d1" + }, + { + "utc": 1795102210.3663821, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.021972459042445, + "trace_id": "952e06945a24b8da03f2767860e6fd31", + "request_id": "d347aa46-ffa5-4ffb-a291-d7233a14a6aa" + }, + { + "utc": 1795102210.869301, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0141157080070116, + "trace_id": "c6173e5bf1f9b577758eb184ccfe6738", + "request_id": "83cd2d89-19bf-4980-b61e-9402e1d55fe5" + }, + { + "utc": 1795102213.36496, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02805929200258106, + "trace_id": "bb2f58a42df159ef66ee7283d1387ae4", + "request_id": "c6ff2437-e905-43e5-9e55-9cdb5b55fa34" + }, + { + "utc": 1795102211.370034, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0253708329983056, + "trace_id": "2d71a3a8d660ce847e88c62589f79419", + "request_id": "343043ca-2d9b-4ff2-a094-0f9886f319b5" + }, + { + "utc": 1795102211.8636239, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.007210249954369, + "trace_id": "fa1c0d268b2484949c261f59f765b397", + "request_id": "9a0a317a-f054-409d-97a6-0099963ffeaa" + }, + { + "utc": 1795102212.371409, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.016411624965258, + "trace_id": "f1ed8672a740c693ee4737f103e3341b", + "request_id": "f530376d-12fa-49aa-84df-165c33d557e8" + }, + { + "utc": 1795102212.873415, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.008516540983692, + "trace_id": "97a0a165015f91d02468d302cf45ce6f", + "request_id": "6c76e7ee-62c7-4e94-bf83-c70bfa9288f1" + }, + { + "utc": 1795102213.364942, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.016758041048888, + "trace_id": "477932102d2b881b4ab70b2d7871f6c0", + "request_id": "d7c9d060-2885-418d-9cdd-6a282f940797" + }, + { + "utc": 1795102215.37352, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03359241702128202, + "trace_id": "f71fde045e993e2ed3558006689dc331", + "request_id": "a11700ef-650a-4ad9-ab52-282725f3b692" + }, + { + "utc": 1795102213.870152, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0086884170304984, + "trace_id": "5d07d961f75b6b8fea5a71cc82ca80ab", + "request_id": "ee31160f-f454-49d4-bc6e-3336b288820a" + }, + { + "utc": 1795102214.3734012, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0216577079845592, + "trace_id": "5f0915b4af5659426b67bf1c36fbb3ef", + "request_id": "f59792ef-1f3d-4fbb-af6c-6b0ac88b057f" + }, + { + "utc": 1795102214.8733592, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.011039584001992, + "trace_id": "ccdc4ad4898fa09893ae1f2d76783e4f", + "request_id": "5e0e11e5-fe04-4d77-8cd3-5df55604cb81" + }, + { + "utc": 1795102215.373479, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0197627499583177, + "trace_id": "c4ea981ca884d1365d4ab1a23ff6d8ec", + "request_id": "d8f7b62d-40bc-43d9-84a4-ed8dc85a3e99" + }, + { + "utc": 1795102217.369165, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030773459002375603, + "trace_id": "ed81eab11807a18e46d0ebfd2980aea9", + "request_id": "b37f7fef-64dd-431a-97b1-2d169d4c1467" + }, + { + "utc": 1795102215.863764, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0066800419590436, + "trace_id": "ca442146ff9d06597d4c6a576aff22d3", + "request_id": "07da0246-4531-43bc-aeaf-931147c07615" + }, + { + "utc": 1795102216.36938, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.017998458002694, + "trace_id": "8d72cf1a42ae9dd2dfecfdb74dbcf2d5", + "request_id": "12cd6add-c086-4f41-8120-f532573d862a" + }, + { + "utc": 1795102216.871444, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0206755840335973, + "trace_id": "0efa3e823afa946cbe3bec9d20ec1281", + "request_id": "c6fb5e47-db2b-48f4-9d40-47d510298c61" + }, + { + "utc": 1795102219.373457, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.014651958015747368, + "trace_id": "f82636045034fad1305a776ed2e23d4c", + "request_id": "79db98d0-63c3-49ed-a79d-1baf27dbccd8" + }, + { + "utc": 1795102217.369136, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0192195830168203, + "trace_id": "c2b1cfe59cd7cf21998d4b1786f4c29a", + "request_id": "91efe735-6ce9-461c-a2a2-0bbf89e027dc" + }, + { + "utc": 1795102217.871538, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.005880833021365, + "trace_id": "8ce05556ca1370c0ab7b552276467258", + "request_id": "5e5c4903-20cd-4cfb-acbb-526d58c85515" + }, + { + "utc": 1795102218.371346, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.016228874970693, + "trace_id": "0f90f27f902ee903c33da1780e428ba3", + "request_id": "6b6df62c-c76f-4123-8d0c-133d45411cab" + }, + { + "utc": 1795102218.873376, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.018794416973833, + "trace_id": "95430a66b2e91dc3b84cf268e3f5f75f", + "request_id": "0238801e-c969-4ee0-adea-18e7456b2243" + }, + { + "utc": 1795102221.3641129, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.018012290995102376, + "trace_id": "d8c40c16f82c09e3323e5d2c714d0cc2", + "request_id": "bd877815-cc3f-4052-a04f-baebd00c6362" + }, + { + "utc": 1795102219.373417, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0088000419782475, + "trace_id": "f52808741458248d214de8bd4d9e19d2", + "request_id": "972c97ca-ebd0-47a9-a062-4d6529dcb022" + }, + { + "utc": 1795102219.8700979, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0055275409831665, + "trace_id": "1d1e1305e7af97b1b79eaad5f71e3f67", + "request_id": "f5b0f4e1-9edd-4167-89ed-19db262eece0" + }, + { + "utc": 1795102220.370392, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0189202500041574, + "trace_id": "b32d4afa6eb134f22f746c5559a52a64", + "request_id": "d0e69afc-cb74-42ba-b862-a1981652f5e2" + }, + { + "utc": 1795102220.869379, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.018999166029971, + "trace_id": "52a54285bc85dab2f97993c467de42c7", + "request_id": "d0217e58-dcf3-43ad-9ff9-23c27c153f10" + }, + { + "utc": 1795102221.364097, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0125845409929752, + "trace_id": "e8e7f9b575003941d48e00e284b7d320", + "request_id": "179f8399-0cc6-44a8-bc21-ffb2d7f79b29" + }, + { + "utc": 1795102223.3695, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.023734583985060453, + "trace_id": "c8ff01ee01c741b2755be5c69fc4cf6a", + "request_id": "425721e6-c0a4-4db2-bdcc-2de64da60256" + }, + { + "utc": 1795102221.865536, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0098279169760644, + "trace_id": "91c7925ebaa663375a25189f386b39d6", + "request_id": "40e98f0b-8670-4fca-ba68-621502265ef4" + }, + { + "utc": 1795102222.371369, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.019390499975998, + "trace_id": "3acf24e7bd65d1678937df55d122efac", + "request_id": "baac8b3f-5988-4d1b-9293-a0993685f821" + }, + { + "utc": 1795102222.8693812, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.017637042037677, + "trace_id": "4ec70d85fa92d07ead91508294277fc6", + "request_id": "014aee3f-14af-4de9-95c5-999c7146b465" + }, + { + "utc": 1795102223.369459, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0118052500183694, + "trace_id": "7a86dcf0961dddf5183f8f812ca3a79f", + "request_id": "c769dc85-94ef-4023-ba55-6b3da9dba09a" + }, + { + "utc": 1795102225.372926, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01700616703601554, + "trace_id": "63b409077792edbe742af9fb61b5353c", + "request_id": "82681c3d-111b-4e82-8396-2dd40c62b43c" + }, + { + "utc": 1795102223.871316, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0090875410242006, + "trace_id": "20a25bd3bc2df09d0714511db6979797", + "request_id": "23ddba5e-2811-4201-8ad6-c03ba19ce71c" + }, + { + "utc": 1795102224.3713422, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0182983329868875, + "trace_id": "3ae553f44e5945e3ec6884fac57a6e4a", + "request_id": "6b66f6f4-5ffb-421a-8799-c6f74f61d201" + }, + { + "utc": 1795102224.868188, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0203830410027876, + "trace_id": "b9f36edde9c7ba02a2e983abbc065986", + "request_id": "36dbecfd-13c3-48fa-9486-99efa0254ddb" + }, + { + "utc": 1795102225.3728929, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.008118374971673, + "trace_id": "d2b001d04e35b576fb9fb540b7316047", + "request_id": "c9e72c18-0232-4052-b6de-d70785810901" + }, + { + "utc": 1795102227.369417, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01737749995663762, + "trace_id": "5951213e6168ff3676181ef102f270f7", + "request_id": "3fd8a143-64ae-40fb-aa5d-cfc965509341" + }, + { + "utc": 1795102225.869386, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0101120420149527, + "trace_id": "dbfb247c138ae2a0bab35915e7ec3cec", + "request_id": "82a21ef0-bd17-46de-8441-8bd99801a8ea" + }, + { + "utc": 1795102226.365689, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.015782208996825, + "trace_id": "773753a6fe65a38e02934c4cee88e096", + "request_id": "3a24f79a-10ba-41b3-8ec7-ba78f0844fb3" + }, + { + "utc": 1795102226.871346, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.023615542042535, + "trace_id": "ca4addde260ce1072389951818a07a72", + "request_id": "85df6844-c27d-41fa-b48f-24bccad4cfc0" + }, + { + "utc": 1795102227.369379, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0097634159610607, + "trace_id": "8a11239eac02460a29088ccac23656f1", + "request_id": "8b38f826-f717-47fb-aeaa-de26ee6dc795" + }, + { + "utc": 1795102229.366067, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02994529198622331, + "trace_id": "0e721fee6af17719240e7174cd302d43", + "request_id": "f46a53b6-dff5-4f5f-84c3-1e5767868a99" + }, + { + "utc": 1795102227.873386, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0121294159907848, + "trace_id": "8dd344acf602abf8062e08bda0d4c444", + "request_id": "9cd8bc7d-1b5e-4696-ad85-816caad44fc7" + }, + { + "utc": 1795102228.3677409, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0131652090349235, + "trace_id": "1740e61e33225bddd4f0c093986bf33a", + "request_id": "1b01802a-b646-4d12-8e1e-2128cf5dab94" + }, + { + "utc": 1795102228.8683221, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0215073750005104, + "trace_id": "c48b6c268c9aadabc8d85d03b2abcc43", + "request_id": "37a242dc-3673-41c0-89b9-57f57f9b3aae" + }, + { + "utc": 1795102229.36603, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.01659875002224, + "trace_id": "72b02f36b183bedd401f147dffde11c2", + "request_id": "af5dcf32-227c-42b0-9367-efb9b3688756" + }, + { + "utc": 1795102231.368295, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.050275416986551136, + "trace_id": "540398f7d15ff63164e3bb296c069b13", + "request_id": "d278fe99-71f9-48d8-9317-cfe1bb224c93" + }, + { + "utc": 1795102229.86541, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.006734083988704, + "trace_id": "9ca17bdde9599ca3870829b045281f78", + "request_id": "e5b1d4de-c8de-413f-96c9-9ff9e0450b58" + }, + { + "utc": 1795102230.368304, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0240850829868577, + "trace_id": "f7b10033efa8f05ed28021dc81bed66e", + "request_id": "bb8fbc49-5598-4ac9-a10f-1d47beb13582" + }, + { + "utc": 1795102230.863349, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.016927166027017, + "trace_id": "b48ea14c62565a7125dd89b2ca64e72b", + "request_id": "a94bbe3f-00d4-4ce6-805c-a9385908f3dc" + }, + { + "utc": 1795102231.3682659, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.024188834009692, + "trace_id": "5e8f1630e59ab6a3e9c3b221c996278a", + "request_id": "c321f93f-05d1-4936-814c-84562da745ce" + }, + { + "utc": 1795102233.36841, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03436362504726276, + "trace_id": "d3d18734599a1ee8926ffb371ebf7fde", + "request_id": "c1dd96e0-a936-4a23-9e84-e3349700b18a" + }, + { + "utc": 1795102231.868265, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0076220000046305, + "trace_id": "8daf7e5306bdc3ca6a20ce1155d0218d", + "request_id": "9b5db87d-3129-4049-8207-6a33a5cf8351" + }, + { + "utc": 1795102232.3651671, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0122032500221394, + "trace_id": "41fb2dc0500a1917e64d12948fed1e45", + "request_id": "a499b48b-e5e7-4df8-b0ae-061672e0f84a" + }, + { + "utc": 1795102232.868341, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0230063750059344, + "trace_id": "6d54f0fe037802a535be612b9ef04ed7", + "request_id": "6825b974-b437-4393-92f1-13cd531a0447" + }, + { + "utc": 1795102233.3683681, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0196266249986365, + "trace_id": "fe9475bb7ef86652ae46dbfb308e1e34", + "request_id": "10a8848e-e087-4d50-8296-1540984dd905" + }, + { + "utc": 1795102235.3650298, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.033437916019465774, + "trace_id": "58ad565f9636222f2f989b304e82d906", + "request_id": "d50a0f0b-d384-4b5b-ac54-d937c696fbe2" + }, + { + "utc": 1795102233.868375, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0106826670235023, + "trace_id": "6d7c6cfc7f98eb0021e30b8fb19147a5", + "request_id": "09d3e588-7472-4cdf-abb6-685503bc2853" + }, + { + "utc": 1795102234.363699, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.012539957999252, + "trace_id": "72d9e8a08b280bb3df702f4ba7a1a2ab", + "request_id": "40675710-c94c-4fdb-906c-c40e4358e043" + }, + { + "utc": 1795102234.868333, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.018584083998576, + "trace_id": "d3f04fbae9d973e3f9d490c666298fde", + "request_id": "8fbb145e-8d23-4d57-aa62-977f658f8892" + }, + { + "utc": 1795102235.365005, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0188208340550773, + "trace_id": "b51f8e977a0fd145e31a182cc1fe5928", + "request_id": "13ee8fb2-b8ad-4ba7-ae5b-1dc8a623ceb2" + }, + { + "utc": 1795102237.368387, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.033939249988179654, + "trace_id": "96093326de91bbf9bc3f9466593cae33", + "request_id": "63a184d2-50c8-4ed2-8330-879f3ffb71b7" + }, + { + "utc": 1795102235.868358, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0057108330074698, + "trace_id": "b94254e06806c03416376ee93dfd3bc7", + "request_id": "d246700d-bd2e-44cf-834d-e0b1c5ece4bb" + }, + { + "utc": 1795102236.3641422, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0208462080336176, + "trace_id": "82083d9c41ea79669ed7f41b5dcca683", + "request_id": "84ee43a4-023c-4dee-ac36-ac6b7317125b" + }, + { + "utc": 1795102236.868329, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0181250830064528, + "trace_id": "45e76f6a58be0053ab94d153eded14af", + "request_id": "eb2c31e0-037d-418e-af9c-3e0df46264ea" + }, + { + "utc": 1795102237.368345, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.021344166016206, + "trace_id": "52c63e87b01eea5f57d4a8c10ff203b1", + "request_id": "a6918223-8eb0-4f84-b296-3646e220ec79" + }, + { + "utc": 1795102239.368358, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03431854100199416, + "trace_id": "583d98d0cd52e79ca07dfb69347ed169", + "request_id": "f633138d-be89-4f06-ace7-60f204e97915" + }, + { + "utc": 1795102237.8682969, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0059343749890104, + "trace_id": "6f89d8add6d7f1b871e66861ab8300dd", + "request_id": "481c3d87-bc56-4bf1-8a07-35e767bd3876" + }, + { + "utc": 1795102238.367687, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0222385420347564, + "trace_id": "0185b735fcd0ae0e4a2e736b4f4b8772", + "request_id": "1fbd0250-eaeb-4a04-9bc5-6032d5899706" + }, + { + "utc": 1795102238.8682811, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0155234999838285, + "trace_id": "21474106dae9c158820bb90009e1e76d", + "request_id": "1e367860-39f9-409b-8ace-98c4c96adc0b" + }, + { + "utc": 1795102239.3683178, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0206327500054613, + "trace_id": "d0d2bd1810e8a0a114d1955b28a65af1", + "request_id": "4097b013-f888-48ee-9e42-8f4059c626bd" + }, + { + "utc": 1795102241.368347, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.035820749995764345, + "trace_id": "6c78b934683eb773faee2728c5e2e641", + "request_id": "99ceb828-f699-4234-9d82-dff95a99bc6f" + }, + { + "utc": 1795102239.8682292, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0059020000044256, + "trace_id": "71f8508e844fb5ecd26664ea6d747df1", + "request_id": "120202f7-021c-40e2-9eb1-139ff1a25107" + }, + { + "utc": 1795102240.36832, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.019284875015728, + "trace_id": "074f789bd408678c67da077bb0139750", + "request_id": "ca5b323b-667a-4bcc-b836-4e136d0083b3" + }, + { + "utc": 1795102240.868321, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.00940450001508, + "trace_id": "93e8502e7681bf1cd44938574af0c922", + "request_id": "0886b5cb-5b58-4f1f-87db-b96d9f1ed600" + }, + { + "utc": 1795102241.368308, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.021136708033737, + "trace_id": "2b308925c4d6270bc505da0cde8d8e84", + "request_id": "ab195ec4-b4d7-401b-bbbf-3edb3585cdb5" + }, + { + "utc": 1795102243.366234, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.034237709012813866, + "trace_id": "ac2728d88ee3246911a9456643324936", + "request_id": "e5b67946-33dc-4569-8989-402261013eaa" + }, + { + "utc": 1795102241.8682308, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.00679349998245, + "trace_id": "4e2695aaf3b6ac828bf7d2e5b7def5b3", + "request_id": "00a2664a-3eed-452a-8fa7-0c1641545785" + }, + { + "utc": 1795102242.3682969, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0176219170098193, + "trace_id": "3bc56799495b375886c9663f7f63bb97", + "request_id": "7d6f54ea-f5d7-4fc1-83a7-f164e4ffd633" + }, + { + "utc": 1795102242.868256, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0168629169929773, + "trace_id": "5e532a86f5aae2d2ff58fc6194b73f90", + "request_id": "5e620774-6048-45dc-bed4-93b26fd46be9" + }, + { + "utc": 1795102243.3661928, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.020733915967867, + "trace_id": "96df2851f163d1caaed22e94ee1919d6", + "request_id": "a5faa3c4-b8b1-4bc2-ac67-8bceba70ae9f" + }, + { + "utc": 1795102245.364948, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03338333300780505, + "trace_id": "63cdc1123639e9caa3f6357369f3e1ca", + "request_id": "e108a88b-9e5b-48c9-8a81-11cf1eb53458" + }, + { + "utc": 1795102243.8644779, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0072906669811346, + "trace_id": "56523432214fc2f2ef49eee86db89021", + "request_id": "0d8fb768-8dd5-4e42-a4c7-ddbb496f6c38" + }, + { + "utc": 1795102244.3679419, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.017651041969657, + "trace_id": "f3654c8decd24813f41bfb6b3b14e540", + "request_id": "15661191-2679-43b0-82a7-e8334d3f9812" + }, + { + "utc": 1795102244.8640482, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.017776374996174, + "trace_id": "b32d5676fe81df31b3658f310655fae4", + "request_id": "2639d327-e02b-4319-8fca-95697147791f" + }, + { + "utc": 1795102245.3649201, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0159357499796897, + "trace_id": "b78a06aacadc890b47ef19f10a700d80", + "request_id": "452c1f0a-ef66-4700-b6cb-d8e887860438" + }, + { + "utc": 1795102247.3683531, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.015532709017861634, + "trace_id": "394d0818d84e1affb6c8034699f87247", + "request_id": "2b1bc0e4-c5ea-4985-89c4-72092b7d94d1" + }, + { + "utc": 1795102245.863631, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0059654579963535, + "trace_id": "025e6f9fb67608e1873431de131b88a5", + "request_id": "1aaa74d9-889f-4510-85d7-6f80638111f6" + }, + { + "utc": 1795102246.3644302, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0185410840203986, + "trace_id": "84fcc39afa684f56bc7b9c4ed5eef3e1", + "request_id": "b0fde0d9-270e-49b2-8b08-5984ce5afde9" + }, + { + "utc": 1795102246.866097, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.0186071660136804, + "trace_id": "aa29f33a70705f1596ed933c893d2086", + "request_id": "5af28896-a669-4d01-aa06-f7248445197f" + }, + { + "utc": 1795102247.368319, + "phase": "sustained_db_lock", + "kind": "message", + "status": 503, + "duration_seconds": 2.008714375027921, + "trace_id": "f124d93f84dd25c5af978e78e2028646", + "request_id": "3c5e5b14-5825-41b8-a227-93cba420a532" + }, + { + "utc": 1795102249.363987, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.016833333007525653, + "trace_id": "d44f5f6f91423989d4e224685271abe2", + "request_id": "718ce308-fa83-40bf-b17a-35de035e02a1" + }, + { + "utc": 1795102247.863969, + "phase": "sustained_db_lock", + "kind": "message", + "status": 201, + "duration_seconds": 1.714371541980654, + "trace_id": "521c6eb4a3b18da362f8f9775be579c7", + "request_id": "5ff1a036-5bd4-4b94-98e1-98f5dc9d1501" + }, + { + "utc": 1795102248.868292, + "phase": "sustained_db_lock", + "kind": "message", + "status": 201, + "duration_seconds": 0.7102389170322567, + "trace_id": "c9e7e19fcdfce8c5a55ab2fb14414c2e", + "request_id": "16038735-ff01-4b30-8db8-6d7f00cdae9b" + }, + { + "utc": 1795102249.3639529, + "phase": "sustained_db_lock", + "kind": "message", + "status": 201, + "duration_seconds": 0.2152367500239052, + "trace_id": "155f861c34c303f4e1934a99e28ece53", + "request_id": "e6448693-d3e2-4677-a9ff-44681091bc50" + }, + { + "utc": 1795102248.368305, + "phase": "sustained_db_lock", + "kind": "message", + "status": 201, + "duration_seconds": 1.2111095829750411, + "trace_id": "e23fc2e2edd97eaf04b35af0b85ea460", + "request_id": "2d70bd61-63ed-4ec9-a8da-bec4331eb735" + }, + { + "utc": 1795102249.863491, + "phase": "sustained_db_lock", + "kind": "message", + "status": 201, + "duration_seconds": 0.007400792033877224, + "trace_id": "fe706df9dbf24d39e65f129693e65ef2", + "request_id": "40a317ae-1aaa-449b-8094-392bb0d6a5eb" + }, + { + "utc": 1795102250.363789, + "phase": "sustained_db_lock", + "kind": "message", + "status": 201, + "duration_seconds": 0.019534916966222227, + "trace_id": "54e806e406b870b4aa55aaa427e7a8f6", + "request_id": "37732b3f-4da8-4baf-b493-974531b7f1dd" + }, + { + "utc": 1795102250.863944, + "phase": "sustained_db_lock", + "kind": "message", + "status": 201, + "duration_seconds": 0.019787333032581955, + "trace_id": "0d5f517a68d37abf463ab6a5445c24b2", + "request_id": "8c4a5db1-1bc1-416c-94aa-3582621db60b" + }, + { + "utc": 1795102251.3661351, + "phase": "sustained_db_lock", + "kind": "message", + "status": 201, + "duration_seconds": 0.023957124969456345, + "trace_id": "4ae89d23153f0ccf698a03e115fe794b", + "request_id": "2d43bd4e-33f1-4fa6-ae2c-f819b254fe48" + }, + { + "utc": 1795102251.3661802, + "phase": "sustained_db_lock", + "kind": "upload", + "status": 202, + "duration_seconds": 0.033413416997063905, + "trace_id": "0344b233dda68abc07b17cc08cf2c407", + "request_id": "e670a440-3a39-48fe-8426-a9ac32fabaa6" + }, + { + "utc": 1795102251.863721, + "phase": "sustained_db_lock", + "kind": "message", + "status": 201, + "duration_seconds": 0.005642874981276691, + "trace_id": "3db2152ba8d15c0cd8834a954b4b7b46", + "request_id": "0867848c-1492-441b-b7fd-9ba04db0b8e4" + }, + { + "utc": 1795102252.364214, + "phase": "sustained_db_lock", + "kind": "message", + "status": 201, + "duration_seconds": 0.016195124946534634, + "trace_id": "87159433df08ab25bec19b8ad4c080e6", + "request_id": "799f6292-b799-4487-ba0b-fd4afd030706" + }, + { + "utc": 1795102252.868247, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.02137966698501259, + "trace_id": "dcd2cc3707df64b296d112ccd5b22879", + "request_id": "d27d5ea5-2a82-4ad5-89cc-b28857a2c389" + }, + { + "utc": 1795102253.3643808, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01881270803278312, + "trace_id": "58c4ea5e37f725f8d8bc3fdbaaf8caf5", + "request_id": "461de3fb-c159-47a5-9eee-c66f2d4bb91c" + }, + { + "utc": 1795102253.3644218, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029788792016915977, + "trace_id": "b6261ce85cb1bc10a52eac02380dc1d0", + "request_id": "1c910ed6-23aa-47df-9343-b02651891d26" + }, + { + "utc": 1795102253.863959, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005490333016496152, + "trace_id": "ab1d7575dcee0d7684cdd7d43677e3c2", + "request_id": "91c1fbc9-6603-4c10-817c-e8993cc36340" + }, + { + "utc": 1795102254.367907, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0184022500179708, + "trace_id": "92cb4f2cf3ef652e6194e132b4c3415a", + "request_id": "56f4c295-eb33-4cfe-9487-d09c30f05392" + }, + { + "utc": 1795102254.864379, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016257334034889936, + "trace_id": "edf42f678a02de8041c583aeeceeae38", + "request_id": "740f3f13-5b85-42e5-ac66-f321e39dfbda" + }, + { + "utc": 1795102255.364297, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.023372999974526465, + "trace_id": "1e7f0b622090a151eec1a1089121d94f", + "request_id": "bb42a27b-7b20-4902-b828-7c5fdd58f1e6" + }, + { + "utc": 1795102255.364341, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.0334431670489721, + "trace_id": "b637d7e0e5d62a9521a01b2e40886b05", + "request_id": "e2ad68c6-9e22-46ac-9e66-b1d8c0b223c7" + }, + { + "utc": 1795102255.864764, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005560625053476542, + "trace_id": "eab844ed9ecf16034d41aa47b040ac46", + "request_id": "af5600a6-63a0-44e6-8558-532c60d6c18c" + }, + { + "utc": 1795102256.3634791, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017507750017102808, + "trace_id": "f35020a0b5866f0400e0cfe24bdde3e2", + "request_id": "b83fa989-18f6-4583-be59-d7d31e31599c" + }, + { + "utc": 1795102256.8682518, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015930832945741713, + "trace_id": "9c83df1cd8e8b14384a97537f9e80fb2", + "request_id": "3e476eed-7a89-4e73-bdb0-7381ef9d2dd2" + }, + { + "utc": 1795102257.364377, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.023102208971977234, + "trace_id": "aa3deac5cf202a3bbdfd2f6c1488a828", + "request_id": "89a281fc-d863-445d-a6e8-b5c571f9380d" + }, + { + "utc": 1795102257.3644211, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.032599582977127284, + "trace_id": "274e393de819298baf54f3e0b3223c2c", + "request_id": "2b915a55-c542-4753-9063-80f1bd045e83" + }, + { + "utc": 1795102257.863242, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005102291994262487, + "trace_id": "504ba93f02c84ebe26ddfe69fc09a3d0", + "request_id": "c00e8a3f-aaed-4e64-9abf-5de045362bbe" + }, + { + "utc": 1795102258.365622, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01744912500726059, + "trace_id": "3af275d9daa174ff96aef18079dd9d4c", + "request_id": "c9daa04b-9a58-46b9-a684-6fec0e78f0a1" + }, + { + "utc": 1795102258.864247, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01835479197325185, + "trace_id": "766a9a1aa36646a570778e8c3b8a5706", + "request_id": "6133ef30-f0fc-4563-bef1-622289236123" + }, + { + "utc": 1795102259.368157, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01863387494813651, + "trace_id": "95a98de96058ad060cfc823cbdbce233", + "request_id": "73141633-fe3c-4cf8-b780-ae85581460b9" + }, + { + "utc": 1795102259.368174, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028904125036206096, + "trace_id": "e94b561bc7e984d1949837ed50bcc987", + "request_id": "d07e4780-03da-4089-8825-1a7aec72d280" + }, + { + "utc": 1795102259.866719, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007484291971195489, + "trace_id": "6cbd153d5270c82f91da8419c3380bdd", + "request_id": "e5baf429-5db9-4280-92fb-02b9da01663a" + }, + { + "utc": 1795102260.3641531, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008434874995145947, + "trace_id": "8ad07b470e66472bbbe2da48695824a1", + "request_id": "68ae1317-5058-4808-8e24-fecba60b1e93" + }, + { + "utc": 1795102260.8647768, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01716599997598678, + "trace_id": "1cddea7d15ebc8947842f3243837d473", + "request_id": "7eeda1a8-12cf-4835-ab85-5d0422ed78e0" + }, + { + "utc": 1795102261.368263, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01868804101832211, + "trace_id": "c722e131963410e7524321fc7608e049", + "request_id": "9c188b15-00c9-4daf-9dbc-3a41ad3a0ea2" + }, + { + "utc": 1795102261.368302, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030324332998134196, + "trace_id": "dc8d777d8e202922591aec2a910617ab", + "request_id": "530a9039-5901-4aa4-96ca-d3efa7f35fda" + }, + { + "utc": 1795102261.8632882, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008341750013642013, + "trace_id": "3db7f9388cdbe3a3834053128756904b", + "request_id": "d3ba9e62-7803-41c5-9f2a-b5ed49e60cf2" + }, + { + "utc": 1795102262.368143, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006113292009104043, + "trace_id": "f46d9c676d77a0b0dbd252ba29fcd4f1", + "request_id": "e0ca2eb2-72e7-45e2-900e-6677ec2a2275" + }, + { + "utc": 1795102262.863431, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016637250024359673, + "trace_id": "9aa4701b67bb433598f32509fecb43b9", + "request_id": "2435ebd5-8b3f-46c3-b009-71b0ded5c0c3" + }, + { + "utc": 1795102263.3682501, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017736334004439414, + "trace_id": "ffbded5f57b139993d9b1d1092d02431", + "request_id": "6c27cab1-31a7-47f9-8b4b-28d3c9f883e1" + }, + { + "utc": 1795102263.368293, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029293208033777773, + "trace_id": "8e0fc6ab9488d09ac27d6e5f77b12f44", + "request_id": "f95bfc61-7306-47bf-a5f9-241d4d5407a2" + }, + { + "utc": 1795102263.868156, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005893249996006489, + "trace_id": "1fc380ec0894302f45150ab511f8772a", + "request_id": "12020e1c-4460-49e6-8fe1-2234571ed98c" + }, + { + "utc": 1795102264.367032, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016282500000670552, + "trace_id": "33865fa3eeb59fe8db2a63e35dca9cac", + "request_id": "3a8cb2ef-17cf-468a-92e7-f285120a5dae" + }, + { + "utc": 1795102264.868201, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0176113749621436, + "trace_id": "8fdfa4f356eebd2873230804e16958ff", + "request_id": "26450adf-32e7-485d-962f-2bf86b8fbc09" + }, + { + "utc": 1795102265.368302, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.019150583015289158, + "trace_id": "bc682cd2f78dfa6696c14598c9e13abc", + "request_id": "b2b2effc-eaf1-4a7c-bc5f-d4f34dcfd1e6" + }, + { + "utc": 1795102265.368342, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030543000029865652, + "trace_id": "e2105c4f5ab2fd8d97be1c398bc89142", + "request_id": "98e2dceb-9119-4131-b954-04c380ae7744" + }, + { + "utc": 1795102265.864158, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008315583050716668, + "trace_id": "ea1fa0f73fbbc2d1bcd5d2f54315405b", + "request_id": "c0161ee4-ddd8-424a-8325-db7667edb534" + }, + { + "utc": 1795102266.368222, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016593041946180165, + "trace_id": "47bda9fb3113f8cf8f4c74f8e5f003b7", + "request_id": "70183f4e-7cf6-4b7e-9e20-2a8795636717" + }, + { + "utc": 1795102266.8682728, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01065837498754263, + "trace_id": "a18a1177eb288f41fbbd0f4f1b8a15e2", + "request_id": "413165d8-0ac2-4afc-8c41-610fd38f3e1b" + }, + { + "utc": 1795102267.368293, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.019365000014659017, + "trace_id": "ff47607d61c9b41b2d83a99862e0c124", + "request_id": "7e314096-4f39-4727-8a94-42acc98e327e" + }, + { + "utc": 1795102267.368339, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030964666977524757, + "trace_id": "b5495d12e63e194362fb3e226158b026", + "request_id": "7c7989c3-9fe5-4bd2-bb14-c254c560dcf9" + }, + { + "utc": 1795102267.8633568, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007460125023499131, + "trace_id": "8c5465e2e34699aaf2d830e5981185ec", + "request_id": "847a247a-d078-49b2-a437-1a4498a079c2" + }, + { + "utc": 1795102268.368226, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01873433298896998, + "trace_id": "d1bc1c4dce59c9be566b66e4e85e4491", + "request_id": "50d4f051-e2d6-4733-b728-8d27588de772" + }, + { + "utc": 1795102268.864257, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006139665958471596, + "trace_id": "a8e85f8f379a6f77da71d085ae6af57f", + "request_id": "094e80c4-4df4-4eaf-819c-8f03e6c57519" + }, + { + "utc": 1795102269.364342, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01978995796525851, + "trace_id": "5959b06e7019ac3fafa2150872238764", + "request_id": "a4d111a6-b3db-452c-90e5-aa3bc5207153" + }, + { + "utc": 1795102269.364383, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030484542017802596, + "trace_id": "e8637ec1ca7cf02ea459df59681ca33e", + "request_id": "1ec7609f-7310-45dd-b41e-b8ae1b7b45bd" + }, + { + "utc": 1795102269.865855, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005458957981318235, + "trace_id": "aee99a2a2685cebb496b60a735512f85", + "request_id": "ead7578e-d1a6-43bb-bd66-1dd9cc418161" + }, + { + "utc": 1795102270.36819, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01703020802233368, + "trace_id": "4324c66e30d54ef9300355cc3348f7b5", + "request_id": "88327e57-f08b-4e05-abab-1e2b71e93d93" + }, + { + "utc": 1795102270.8664892, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015582584019284695, + "trace_id": "4a07cd889dfd8e93b07f5b4247170093", + "request_id": "d4058d5b-81d1-48cd-9423-c1234f8f8f2b" + }, + { + "utc": 1795102271.368227, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017534666985739022, + "trace_id": "cc90c34797e4c8499a5852cf7fc27d82", + "request_id": "3fca3cd6-de6a-472c-a033-0f4e2f0e58f5" + }, + { + "utc": 1795102271.368267, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03007179102860391, + "trace_id": "146156446c42a5f7fede1a3e608ce95b", + "request_id": "c7d30ed6-a09e-4a0c-a227-e1cf464237a1" + }, + { + "utc": 1795102271.86562, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006418665987439454, + "trace_id": "b6d4848a0031f3070a8ba4e51b036ea8", + "request_id": "b0ba3671-8443-4360-b892-fe6739151d60" + }, + { + "utc": 1795102272.367, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0162362499977462, + "trace_id": "9d502eab32734326c1a3c24a0a8cb7d8", + "request_id": "ce3076ac-dc78-40a1-933f-9b5e15b0b334" + }, + { + "utc": 1795102272.866265, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017444458964746445, + "trace_id": "38ecefada9fe1defb7337efa79f6bbcf", + "request_id": "210dfd2b-914c-4c9e-b5e8-492361c0d2b0" + }, + { + "utc": 1795102273.36332, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017640790960285813, + "trace_id": "5b9ed53645982d0907a264c956492fee", + "request_id": "c3f4b8cc-7e6b-4d19-ba88-6616e0b038ae" + }, + { + "utc": 1795102273.363349, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.026702416012994945, + "trace_id": "82f01ba112210f3e66903e99cb55d33f", + "request_id": "60f450b5-3ccc-486c-8596-12ee114312c7" + }, + { + "utc": 1795102273.868114, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0061300419620238245, + "trace_id": "708c46bc17ab97f944e2a0f49ed8b48a", + "request_id": "d1e0e379-606d-461d-bf64-565186a3ec65" + }, + { + "utc": 1795102274.364833, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016534333990421146, + "trace_id": "8440af62852ec0aeb7320098b08355c6", + "request_id": "48a12b75-173b-45c5-a251-e0b130937559" + }, + { + "utc": 1795102274.8640058, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.012726166984066367, + "trace_id": "5a9cb91742e7611d04ad6892ac152c01", + "request_id": "7c4e11c2-03e6-4b3c-ac2b-3b14ca4ce3bc" + }, + { + "utc": 1795102275.368146, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007602875004522502, + "trace_id": "6b6e052b627514671ad165cef2a526fc", + "request_id": "35c91f49-39e9-42ef-8e8c-be3a5c051637" + }, + { + "utc": 1795102275.3681889, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.010708542016800493, + "trace_id": "c83392622f7309a2b9402bc389e13be2", + "request_id": "c5640acf-9b1b-4954-9e34-2bf9667ca082" + }, + { + "utc": 1795102275.865185, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0076004580478183925, + "trace_id": "4ae2d58858425fcb82d3cd46eb12cf48", + "request_id": "575b1ff3-df02-4e6d-8c20-19ca0efbda17" + }, + { + "utc": 1795102276.368255, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.010644958005286753, + "trace_id": "6bf2dd6f9ba9fd754dd4ff17d2500140", + "request_id": "52429818-b039-49d9-a473-211adabe1c0b" + }, + { + "utc": 1795102276.868191, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.010681584011763334, + "trace_id": "a6bd0b5ca867275ea54be6ffd4aea1f8", + "request_id": "5716fb7b-9174-4fc6-a96c-8ac5fe53f217" + }, + { + "utc": 1795102277.3659909, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013852166011929512, + "trace_id": "cedf143d5fbec7a64c6f1b8e6305bc06", + "request_id": "b205c670-034a-4579-b726-71808f42dea9" + }, + { + "utc": 1795102277.366017, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02058908302569762, + "trace_id": "0b49e54a90abd6f996a56db5ccb2b780", + "request_id": "d1d4e4ae-1000-442c-b386-b32c2f2282c7" + }, + { + "utc": 1795102277.868205, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008580207999330014, + "trace_id": "426787bda959cf42a8918d9104d12065", + "request_id": "0149110f-5040-4021-87ea-f7cfc7413d6b" + }, + { + "utc": 1795102278.3655858, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006535375025123358, + "trace_id": "3eea61e07a3d2e30875fe231edad46dc", + "request_id": "988b7a6a-be70-4297-ab23-6b5004ad68f0" + }, + { + "utc": 1795102278.865912, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017103624995797873, + "trace_id": "4ef7157dad976808672ec61e114aad1e", + "request_id": "1f68a48e-e374-499a-9e4c-e600bfc10247" + }, + { + "utc": 1795102279.364309, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017980665958020836, + "trace_id": "9e3c80b9684ef19d3709f939b529cef7", + "request_id": "12f4e538-39fa-428f-8e5f-057a032473f3" + }, + { + "utc": 1795102279.364365, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03055941703496501, + "trace_id": "9eb6512483fca23d2b1d01ae9b9bb8b4", + "request_id": "9c8cbf24-d09b-418f-9cba-fc6608b3cd67" + }, + { + "utc": 1795102279.868194, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007040625030640513, + "trace_id": "f5794168fab3dbfa65fa1e5403af4a74", + "request_id": "1ed53e42-17ee-4786-b364-2234aaea5d0a" + }, + { + "utc": 1795102280.363999, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01637433300493285, + "trace_id": "4a1d31194aed52f3286bb770fe25ac8a", + "request_id": "0c7ec76d-824b-4a7d-bcbd-31f4c2bc3adf" + }, + { + "utc": 1795102280.868242, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.026554499985650182, + "trace_id": "f5059a4831192885a9be6e9cc00c50f1", + "request_id": "525ae4d0-9473-4b92-a971-9a6e42fb5cba" + }, + { + "utc": 1795102281.363955, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018213583971373737, + "trace_id": "e72975d72b3a83ed40c878e37a8f159e", + "request_id": "fd1524ca-76fa-480a-a978-7e418213ee60" + }, + { + "utc": 1795102281.3640351, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03026450000470504, + "trace_id": "2b5f395a0ee4b7209d4e5442fcd9dc99", + "request_id": "c1e133e4-d098-4d2b-8569-c9a5ccfc40ea" + }, + { + "utc": 1795102281.868138, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006581542023923248, + "trace_id": "2fd7c32e53668189b4054c9d653beec1", + "request_id": "c03704ad-0e49-47d9-9487-5cbaa99ef742" + }, + { + "utc": 1795102282.3642712, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016938290966209024, + "trace_id": "3cce6bea98f838b73a743139c10dd282", + "request_id": "9f9529d3-4f33-4473-bc37-e7a1b11fb97a" + }, + { + "utc": 1795102282.863499, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017055207979865372, + "trace_id": "06c9d581297e1703b00924e5215e3761", + "request_id": "7deb6dbe-1a8e-4dc7-89ea-75155cdf9544" + }, + { + "utc": 1795102283.363957, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018776541983243078, + "trace_id": "6b6a91d2f1e8c6e0a0b9a47c6b9c03dd", + "request_id": "a1647240-bec6-4a1b-8dbc-34109d7b6c82" + }, + { + "utc": 1795102283.364219, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030998957983683795, + "trace_id": "c44db4e2ab2471411c260c2d43bbf8ff", + "request_id": "241ce2c7-c8a3-481e-a90d-599990dd8d31" + }, + { + "utc": 1795102283.8680818, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007007167034316808, + "trace_id": "6ea52bc09350f0a7e75aa6f42201b690", + "request_id": "002c980a-ce63-4124-9812-86023f409a80" + }, + { + "utc": 1795102284.365953, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013878999976441264, + "trace_id": "063ed19a5a44e468dfef0ce052dfaa39", + "request_id": "7001c25e-5768-41c7-aa79-dc433ba45bc6" + }, + { + "utc": 1795102284.868174, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.014697375008836389, + "trace_id": "52e01016f968ba5883962cc91e4873a7", + "request_id": "ef0532d7-c905-4dfd-8b62-5671262fdeee" + }, + { + "utc": 1795102285.3681731, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.020112874975893646, + "trace_id": "b021ba6712edd2ac640e393aafcc48ce", + "request_id": "8e01e56d-66fc-4f7b-a457-f7d996123390" + }, + { + "utc": 1795102285.368215, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03148704202612862, + "trace_id": "0ef8d2ebb30972099f13330c0dfe6ed4", + "request_id": "5a70ee61-5d1f-444b-941c-52951649a01c" + }, + { + "utc": 1795102285.868073, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005249625013675541, + "trace_id": "a2713afc4ddb3c6c8e1d9476abfffb6a", + "request_id": "c6a3afbe-d7bb-4d52-b280-efa096f3b190" + }, + { + "utc": 1795102286.3681412, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009228084003552794, + "trace_id": "54137de6a95ea44303a56e80ec1b2302", + "request_id": "663e1b94-8b15-4a68-8b03-5e138d5a6c86" + }, + { + "utc": 1795102286.864188, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009818874998018146, + "trace_id": "1b3e9ef75218775d5e4d689eef35d09a", + "request_id": "f7e7dfe6-c33c-49c3-8010-3fc23c25e55d" + }, + { + "utc": 1795102287.3651042, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.02348874998278916, + "trace_id": "2c1a6bee0d5a47947fdd398a5a14fd73", + "request_id": "3cddc46c-40b8-4beb-97f8-b27740c4aba4" + }, + { + "utc": 1795102287.365144, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03262470895424485, + "trace_id": "87105b694b74ea2a751126359a1e73f7", + "request_id": "8887f11d-4150-4b79-8387-6ef7ea812aab" + }, + { + "utc": 1795102287.8641741, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006524124997667968, + "trace_id": "c66adb37e55854df4b7653b8c326dc55", + "request_id": "0ad64299-4fe9-4429-8392-78264208dade" + }, + { + "utc": 1795102288.363554, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0063272500410676, + "trace_id": "e6c9f2761309d8849936fe5f997b4721", + "request_id": "180e1443-dd8b-4a01-9c82-5f1442840e74" + }, + { + "utc": 1795102288.868145, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016021166986320168, + "trace_id": "df12c0b0a4f5ba0b56f0491a9e1056ef", + "request_id": "d837f1ab-8b4a-460e-90ae-e5369f766940" + }, + { + "utc": 1795102289.368122, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.019528708013240248, + "trace_id": "5f277f7e0b587fabd3cca0a73f932e06", + "request_id": "0fee9f94-c173-47ce-872a-7169087b0bb2" + }, + { + "utc": 1795102289.3684978, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03026749996934086, + "trace_id": "891cd44eaf2f6ce8cd1aa66b66204eb0", + "request_id": "1773b041-4dcf-49d2-8ba0-168c79376250" + }, + { + "utc": 1795102289.863991, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007734708022326231, + "trace_id": "1c5c20003cf89fc7f413d627fb87bf4f", + "request_id": "50eed66a-a8a2-4941-a924-042d46fbe323" + }, + { + "utc": 1795102290.363791, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009875165997073054, + "trace_id": "bcd8975ae9354f58cd84c400169f18d6", + "request_id": "56e55773-d045-496d-a02a-3fef28b37e15" + }, + { + "utc": 1795102290.868136, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.010330290999263525, + "trace_id": "15bc47a1658af2f775272d34ee88fbe9", + "request_id": "52b34ba9-5ebf-4fc3-b9f6-3b20ebd74dcf" + }, + { + "utc": 1795102291.365052, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.027208167011849582, + "trace_id": "e892f44049090cc36bb322582351d99a", + "request_id": "c1b3672a-5bc9-4e51-a254-02202326d76f" + }, + { + "utc": 1795102291.365091, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.034378624986857176, + "trace_id": "5ef12accf5b6dc879e9cb1c227ce00b3", + "request_id": "8fb63276-7d21-462a-8b4e-a622eb2b0266" + }, + { + "utc": 1795102291.864044, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006892999983392656, + "trace_id": "1b29bab77bfa8efd833a50f6b2c583cb", + "request_id": "797348ac-de96-4ae6-801f-28f1229f622e" + }, + { + "utc": 1795102292.363987, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018096959043759853, + "trace_id": "d1a16cb349b3c5ec163d2bb116d3c405", + "request_id": "b6b45f2d-395f-482e-9db1-7a7e4f1a4796" + }, + { + "utc": 1795102292.8644822, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009849624999333173, + "trace_id": "d1f62d5aee62086ee5a8fa495c2bfdea", + "request_id": "ca41525e-aa33-452f-be6b-6641c975db96" + }, + { + "utc": 1795102293.368188, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018091749981977046, + "trace_id": "9b082d5b6bcd92c308b7696589b54e54", + "request_id": "bf479296-309e-42f0-8cb7-efedb42fac6d" + }, + { + "utc": 1795102293.36824, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03102029097499326, + "trace_id": "7960bdb94982767fb6bc8d8a0d393bd9", + "request_id": "01285c1f-f323-4b4d-b523-3f70da508055" + }, + { + "utc": 1795102293.868054, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006094750016927719, + "trace_id": "86a5f4d61c10e875027c770da82fabb6", + "request_id": "144b1e10-1099-49af-bdb5-fb2bd6f9e4be" + }, + { + "utc": 1795102294.368128, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01639208401320502, + "trace_id": "3ef76a8fd46c532aa920886eb6e12d37", + "request_id": "82d301c8-f361-427e-bcdd-e850c38b23af" + }, + { + "utc": 1795102294.8676438, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.014850250037852675, + "trace_id": "2a40d3d1109d9822dfb093fe7f93a14c", + "request_id": "fc8b1299-2198-46cc-8be7-d528feafe0f9" + }, + { + "utc": 1795102295.364147, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018297874950803816, + "trace_id": "8398661993b052507a5683ffd3fe0904", + "request_id": "057ade41-7dcf-4b88-8de1-3e1684c4554e" + }, + { + "utc": 1795102295.364188, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029381000029388815, + "trace_id": "5978546cd99a241ca7fda0cb793dd596", + "request_id": "92331954-550e-466e-93e5-72e8e2ad572d" + }, + { + "utc": 1795102295.864164, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00517187500372529, + "trace_id": "f9f7af4840d32a4fdfd85aeb8c633847", + "request_id": "b808a134-60da-4454-805f-4a16e58f95fa" + }, + { + "utc": 1795102296.3633852, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.020714749989565462, + "trace_id": "357422e4bd869b032d25fac6976ab29e", + "request_id": "bf0f28d5-a636-4e62-a825-aed1f4336141" + }, + { + "utc": 1795102296.864216, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0065768330241553485, + "trace_id": "f739b0960441b6e2814cda08fa93822e", + "request_id": "b5b0b507-0711-48be-b424-6a11b7582a33" + }, + { + "utc": 1795102297.368119, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018765457964036614, + "trace_id": "1865e2c3ac81a687c01f76b52688e73d", + "request_id": "e4c4346e-7288-4002-ad1a-97cd1d7dc817" + }, + { + "utc": 1795102297.368164, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03135479200864211, + "trace_id": "ef5c235442e15acd6604d6320cbf8a6f", + "request_id": "ee474546-9cd5-4ec1-a0a2-13d91f8ffbc1" + }, + { + "utc": 1795102297.867979, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005235165997873992, + "trace_id": "959ed6868ed2c7e45dbe1c3ea81d8f76", + "request_id": "b126b053-9085-4d5b-8c2f-16fb6669854a" + }, + { + "utc": 1795102298.368099, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015597417019307613, + "trace_id": "f329b7f64f47949425471027dedf583f", + "request_id": "a6bcad54-a5cf-41aa-a5d2-f750dd50aaba" + }, + { + "utc": 1795102298.863168, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007697499997448176, + "trace_id": "9762a6922ac672a79208833e6f3ab842", + "request_id": "4171111a-ac47-42b7-9459-b1eaab6d2a3a" + }, + { + "utc": 1795102299.3640552, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0179314169799909, + "trace_id": "9c09214eebb745f2076d06c54b1677f4", + "request_id": "8b3b974e-e6ca-4e99-8f45-338f3c4a3830" + }, + { + "utc": 1795102299.3641012, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02990600001066923, + "trace_id": "af97fb0386eb33d765e8269c1859bab2", + "request_id": "8b0b6184-6508-4b00-9c01-09a4abe09792" + }, + { + "utc": 1795102299.86722, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006080583960283548, + "trace_id": "c2674774d359e5d2fe0123c4ee9a4ef0", + "request_id": "da4be104-fe8f-4b19-9dc0-c64c73bc478c" + }, + { + "utc": 1795102300.3646138, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017488832992967218, + "trace_id": "096d91f7676d2f839ba2190633d5b015", + "request_id": "bd2c07ed-ef75-4585-b87c-db71221648b0" + }, + { + "utc": 1795102300.863408, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008850625017657876, + "trace_id": "85da6eb76e845e2b3ec6808188203777", + "request_id": "aed0a537-effb-40c1-99db-77e860b46dbd" + }, + { + "utc": 1795102301.368171, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01677975000347942, + "trace_id": "77933603620625267092aa2b44968d8a", + "request_id": "b3ac544e-0c88-4737-871c-88a8604872e3" + }, + { + "utc": 1795102301.368211, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02927487500710413, + "trace_id": "8a3de39af8fb0516f9beec8ef4c8da06", + "request_id": "231a2b1e-f7e5-4f3d-8c0f-7a3b15105386" + }, + { + "utc": 1795102301.8636448, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00907591701252386, + "trace_id": "2171b96df1d6faef3b71459de352d277", + "request_id": "be5509d9-3a90-4620-ae36-3c6c0725cb91" + }, + { + "utc": 1795102302.368099, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017966499959584326, + "trace_id": "00de4730a261a43148bbbce6e29e942d", + "request_id": "66badd28-c393-4b4f-8d73-33b8c9250926" + }, + { + "utc": 1795102302.867748, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.02081795799313113, + "trace_id": "a84d09a8380fe0afd14d489bead0e467", + "request_id": "fad3d779-ddf6-42db-9cc3-dbdb8775d992" + }, + { + "utc": 1795102303.364203, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.010453958995640278, + "trace_id": "df62e9b02630e76ac2994b9559ba744c", + "request_id": "fcd4c5e3-aecf-429c-a241-1ed3f9537b4b" + }, + { + "utc": 1795102303.364249, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01738483301596716, + "trace_id": "35bb83dd55a449b1c64646a0cc9f39b3", + "request_id": "97cab4c5-05a3-4927-b9e6-2e51f0c6fb68" + }, + { + "utc": 1795102303.868075, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007238166988827288, + "trace_id": "3d3c64408dd9fde60d1884b4dfe44533", + "request_id": "895606a1-483f-4324-813b-5f9d73e74bed" + }, + { + "utc": 1795102304.3680341, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01878933300031349, + "trace_id": "f1ff1a68bca5a0aad7922087ce290daf", + "request_id": "60b5d521-4c76-4b38-b18c-49efd0bf9bfd" + }, + { + "utc": 1795102304.865346, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013725958007853478, + "trace_id": "45e6f977dce211ac84d09a411fab2a3a", + "request_id": "7e922d57-1ca0-4723-af8b-3365334e5834" + }, + { + "utc": 1795102305.36484, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006433750037103891, + "trace_id": "b85d95cd1b441d230e7af9f8357f8fdb", + "request_id": "6bdfe8eb-316e-4760-8e4e-2f0a50eb1f98" + }, + { + "utc": 1795102305.364871, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011859291000291705, + "trace_id": "a656da2bc71207bfe113dc3cf6170b04", + "request_id": "af520a03-33ac-4c28-a96a-4cca2804ee8d" + }, + { + "utc": 1795102305.8638499, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00679583300370723, + "trace_id": "aae32a700cf34c67832c9cab37f0eaf0", + "request_id": "d7e81111-b103-49ee-ac1b-3d533cc8e910" + }, + { + "utc": 1795102306.366446, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01659391599241644, + "trace_id": "8113d16f5dd23dda2f7d2402ee15b655", + "request_id": "fc728f94-259d-43af-9b34-ac0434de31d8" + }, + { + "utc": 1795102306.868047, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.011634666996542364, + "trace_id": "10f222af9f7e0cece5ecefa17415ee9d", + "request_id": "1ee1861a-f9e4-4654-b8af-7683705b7aff" + }, + { + "utc": 1795102307.3639529, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01391787501052022, + "trace_id": "e8bfb34be097532427ce2b0b34e5c77a", + "request_id": "7a1920a5-b8d5-418c-a87d-7b1aebea103a" + }, + { + "utc": 1795102307.364008, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02261770801851526, + "trace_id": "802cec72b629beecaa172b380c922dd1", + "request_id": "ab13359d-e11c-47b3-a3db-a399e96d556c" + }, + { + "utc": 1795102307.8660328, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005410832993220538, + "trace_id": "c8b099ca55aca2d26ea1a5d8e7c41438", + "request_id": "3ecfa467-7a2a-4102-8bf6-d2aa88b1150d" + }, + { + "utc": 1795102308.365396, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017452041967771947, + "trace_id": "6f76cbab67483070b4e6af0af7d8ede3", + "request_id": "3d4d3ffa-e186-49d6-b0e5-27b327f2bb11" + }, + { + "utc": 1795102308.863868, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017990542051848024, + "trace_id": "e3fc777c66ed5b4e7268221889a159f8", + "request_id": "2788830b-467d-4e41-bf15-a9ee5a174038" + }, + { + "utc": 1795102309.3671181, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009756207989994437, + "trace_id": "8ceb66abb110b3254fd70b1e3d601707", + "request_id": "abee0d93-f836-4312-a7cf-ec05b3036663" + }, + { + "utc": 1795102309.367161, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.016245124978013337, + "trace_id": "fc5792647f203caa70fe36f42d98f394", + "request_id": "5d952246-543e-43ad-8276-dcf2a9d12de6" + }, + { + "utc": 1795102309.868007, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005605958984233439, + "trace_id": "76816b1dbb2ff427ec04f03d9acac6ac", + "request_id": "1c9355fe-b464-44ca-b053-c518fe6ff131" + }, + { + "utc": 1795102310.3681228, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.012226334016304463, + "trace_id": "1d159de8a15acf0d969c1630a8dbe292", + "request_id": "bce292d4-8ddb-42ae-8d73-e6d47c22298b" + }, + { + "utc": 1795102310.8680909, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.014322082977741957, + "trace_id": "664c4674572a0d56265ebc059061161d", + "request_id": "924d236b-3fcd-4c73-ae78-ea07bc5df675" + }, + { + "utc": 1795102311.364129, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015899083053227514, + "trace_id": "f5a7509eece7ab43d7862d4e81685313", + "request_id": "ccc891f1-05af-4715-bdad-cb1e9ae027d0" + }, + { + "utc": 1795102311.36416, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.024637875030748546, + "trace_id": "5c4705c27b694c5a13c632dfaaacd5fc", + "request_id": "57882fa2-173c-47f2-8796-cd6e4138599b" + }, + { + "utc": 1795102311.8679838, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005050000036135316, + "trace_id": "7f97dcb33057171d43688ff126204870", + "request_id": "b0b57629-c42c-44fe-9613-6c0200bf78f1" + }, + { + "utc": 1795102312.3658922, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017810082994401455, + "trace_id": "11eba944fad9531826546092cbe71f28", + "request_id": "c9e8162c-c6e3-4594-940b-6c742f9c82a5" + }, + { + "utc": 1795102312.864332, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008190041000489146, + "trace_id": "dce1082610e4a4de040d99f628eec0d5", + "request_id": "315d7c64-c8b6-4e9d-a80f-c5857f5daf32" + }, + { + "utc": 1795102313.366904, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016204959014430642, + "trace_id": "5934e3587e4018dd1c67799dfdf1eaa1", + "request_id": "9f1dc0c0-27eb-4e56-ad62-6b3db9af2ab6" + }, + { + "utc": 1795102313.366951, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.0295757909771055, + "trace_id": "e2baf15c7c67a89eed019cc84c42b359", + "request_id": "7a611985-ef7a-4523-a054-ba066234843d" + }, + { + "utc": 1795102313.863586, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006994042021688074, + "trace_id": "2fe1dfc8e70c352a9750584eb024caa2", + "request_id": "5521d4cf-9cab-4875-ac64-c40a87ae186e" + }, + { + "utc": 1795102314.3680582, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01614500000141561, + "trace_id": "1461a12c31fd42d9aef4cff7cb06555a", + "request_id": "300e866e-e295-489c-959a-99340592e52d" + }, + { + "utc": 1795102314.865749, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.014484417042694986, + "trace_id": "d3d299eea66b2eb34734a4c2e5a672f8", + "request_id": "01e426d4-a7cc-444f-b347-50374eb44c33" + }, + { + "utc": 1795102315.368055, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01705341599881649, + "trace_id": "61bbd7140918d975e6c562102bf8ed31", + "request_id": "e74af4ab-3198-454b-90fb-8de4e5f1ab8a" + }, + { + "utc": 1795102315.368099, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028143249975983053, + "trace_id": "046dac52a37b82151b2e8f07130a7b5c", + "request_id": "a40f7f77-8702-4748-a77c-52f81318f087" + }, + { + "utc": 1795102315.8680708, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008276749984361231, + "trace_id": "28e1ac981154ed7b15df8311e29b48d1", + "request_id": "b49640b1-4890-458e-9a6e-6fbc8a1d8c8e" + }, + { + "utc": 1795102316.3634639, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0174981250311248, + "trace_id": "872f1df37186286cd7e691c998e9c872", + "request_id": "720577d7-2e10-4e2a-b894-2de33c23b5ee" + }, + { + "utc": 1795102316.864574, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01721720799105242, + "trace_id": "2c6320f39c52b8e1b5743e63911bf537", + "request_id": "c47df42c-420d-4967-bfa9-3761ff007af6" + }, + { + "utc": 1795102317.36808, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.020085583033505827, + "trace_id": "2edf3051c070154e77e2128dac644fdd", + "request_id": "3e6fb51f-2059-42c8-b290-69ac0866e447" + }, + { + "utc": 1795102317.368125, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03189233399461955, + "trace_id": "036be5f084502afffcf70331f26a4ab3", + "request_id": "9f9f2512-4874-4521-920c-f57394e371f9" + }, + { + "utc": 1795102317.8664598, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006136749987490475, + "trace_id": "9056570f92f199b7fd47f094a904d3f2", + "request_id": "4addab71-820b-4ca0-9364-aaadabcaf88c" + }, + { + "utc": 1795102318.368057, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008077291015069932, + "trace_id": "83db8fc6924e2ebd98f08524015d8254", + "request_id": "19c16aa2-182c-4e6e-94a6-2829dac03bc8" + }, + { + "utc": 1795102318.868051, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01623266702517867, + "trace_id": "d3373b49568724c6c7872ec62c976883", + "request_id": "1c5280fd-1cc9-4154-b804-ce839f7aa7cb" + }, + { + "utc": 1795102319.368065, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016259875032119453, + "trace_id": "f7c5e4cf18a0ecf04b73b6950dfa9b53", + "request_id": "fa8d3139-8fb9-4155-a53f-93394b8ca59a" + }, + { + "utc": 1795102319.368107, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.0292304580216296, + "trace_id": "ca73910801cb50db245ee73e71574302", + "request_id": "a9c4a196-cb83-4c3c-97d5-5fb0234d4f79" + }, + { + "utc": 1795102319.865514, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006130666006356478, + "trace_id": "cac57536c896428b98fe5d52be93124c", + "request_id": "8f3cebbd-580a-4f5f-88f1-32b1c17ddcdf" + }, + { + "utc": 1795102320.36797, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006989583023823798, + "trace_id": "462eda7e709514cf2cb5dcc617b0d54e", + "request_id": "808603c3-2cd0-43ae-bb10-a4a17d23b034" + }, + { + "utc": 1795102320.8653178, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016410999989602715, + "trace_id": "c5c9463b6617221f33cdb166326bd1ba", + "request_id": "38281343-14d8-47c8-b21e-ba87d5e9467a" + }, + { + "utc": 1795102321.364562, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01739183400059119, + "trace_id": "430365c1dbe1ec0549159081dddaa067", + "request_id": "d20a8393-c073-4235-9e1b-9aa4e6d5214f" + }, + { + "utc": 1795102321.364602, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02944020798895508, + "trace_id": "112607a32f7182fe0c10a518b789fc99", + "request_id": "6034d896-f60b-4719-8389-fb508e46f6ec" + }, + { + "utc": 1795102321.864523, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0053435409790836275, + "trace_id": "90ee981034b6b2340906caa8c3e0e804", + "request_id": "95e38ef6-5f62-4de0-a17b-5e49de4f7fad" + }, + { + "utc": 1795102322.36433, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017584708984941244, + "trace_id": "77ff9e6925d23ff7a0666c46c14c72f8", + "request_id": "51f81fbf-3d9f-48f5-8e0b-d5553dcb9123" + }, + { + "utc": 1795102322.8663402, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01628295803675428, + "trace_id": "a3d3e176da86483435ea0745ef68537c", + "request_id": "f83ac6f9-5b02-49c7-9e8c-0251deb2502d" + }, + { + "utc": 1795102323.3673291, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015168041980359703, + "trace_id": "84d8d0225787e013701450077d9d508f", + "request_id": "054a3686-ae50-44c1-ac8f-21b16cde4de4" + }, + { + "utc": 1795102323.367361, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02584433404263109, + "trace_id": "700c49d112051fc8b64680295c59f976", + "request_id": "883c2f28-fc14-4da5-b558-1a5a1b522b1a" + }, + { + "utc": 1795102323.867923, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005930749990511686, + "trace_id": "61cbeb512ef61b0f54e9ae3357f684a4", + "request_id": "062c5894-03f0-4240-be53-861baf3a6fc9" + }, + { + "utc": 1795102324.3670158, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009900499950163066, + "trace_id": "0e9d5ee3a7533c40340329670a04d13e", + "request_id": "a94ec00e-6149-4e61-9a30-ea6b939b7c78" + }, + { + "utc": 1795102324.864724, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008583041955716908, + "trace_id": "57c97d189e391553ab5fdf755eeb577a", + "request_id": "e54b505f-443f-4fdc-87e8-abc4d09982a4" + }, + { + "utc": 1795102325.368047, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0183253749855794, + "trace_id": "426f6ddb21029f8d55bfc034f24095d0", + "request_id": "a2d01b2d-090f-46f1-9a34-b6df75b7fd89" + }, + { + "utc": 1795102325.368085, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02942679199622944, + "trace_id": "4303ec2192fedb5e2f56e82487dcb586", + "request_id": "22101de4-fe77-44f7-b113-40547f311119" + }, + { + "utc": 1795102325.864316, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007801792002283037, + "trace_id": "c02fd7bfea6662fef23591e8fa8a930a", + "request_id": "11b3fdb6-ad09-4c50-8881-99fabd6b157e" + }, + { + "utc": 1795102326.368028, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013192749989684671, + "trace_id": "54aa4f38477d9bd2e48c7d08dd8f81b6", + "request_id": "e1590b16-0f58-495d-a1d7-8473312c9a36" + }, + { + "utc": 1795102326.863355, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0070302499807439744, + "trace_id": "e84ba8e8fd555ccd7f638b4324a0db41", + "request_id": "8b2a2857-a40d-496d-9d76-cf765432ab35" + }, + { + "utc": 1795102327.364336, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017567291972227395, + "trace_id": "42522283b7f1d98fbbed279a71cbe888", + "request_id": "1fe484eb-5297-4f07-acab-d7403b2b2069" + }, + { + "utc": 1795102327.364379, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029902958020102233, + "trace_id": "22b88fd5d62c15e3b1b9b767228474a7", + "request_id": "9f908a04-a3f1-4608-903a-445d468fbb9c" + }, + { + "utc": 1795102327.86802, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00717679102672264, + "trace_id": "6034b2ceaaacb4457029ff72b1c2e34b", + "request_id": "924e3322-7278-4899-ab5b-97719e5ad27a" + }, + { + "utc": 1795102328.3660932, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017390166991390288, + "trace_id": "a6b96110897a5c896313f7b6079b8c78", + "request_id": "2945ceb7-f846-46cf-be08-d63404530071" + }, + { + "utc": 1795102328.8649752, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.024640624993480742, + "trace_id": "679c709966131a130cbbb1bd026323c9", + "request_id": "41438a3e-1416-400d-9f15-0aa21cda208c" + }, + { + "utc": 1795102329.368027, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01871458301320672, + "trace_id": "95116ac9e777f0c342edc617b425edd7", + "request_id": "96c0d4d1-00c1-4933-a68d-b160ef3ba3dc" + }, + { + "utc": 1795102329.368072, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03154008398996666, + "trace_id": "6d00831b8b4b8738b0143860b7814d73", + "request_id": "3fd7f69e-7a38-4566-b3bd-76375102cdea" + }, + { + "utc": 1795102329.863329, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005296082992572337, + "trace_id": "2b168ecebd67f790e2a3ee8d6d7a2611", + "request_id": "efc8960c-fdaa-4af1-858f-e6c1f8de09db" + }, + { + "utc": 1795102330.36387, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016352459031622857, + "trace_id": "f71eae6a91d7a8de412aeac50b91ba5d", + "request_id": "effdd160-9ef7-4295-9fe8-c0ea3a055730" + }, + { + "utc": 1795102330.863758, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016790708003100008, + "trace_id": "e8b3fefcf677bd3fe1ae4a2c294e1622", + "request_id": "81735581-a591-40f4-a504-95a948ec8897" + }, + { + "utc": 1795102331.364109, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.010042292007710785, + "trace_id": "23651b755356d645a15909f8afaf2a80", + "request_id": "ea490b21-c0c0-4c96-be3c-e26dc8479738" + }, + { + "utc": 1795102331.364152, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.016392249963246286, + "trace_id": "2fff2824ea902a2e3fc2d1c269f01040", + "request_id": "55cb03fe-0fa8-485d-9653-72d4656c46e6" + }, + { + "utc": 1795102331.8642712, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005079584021586925, + "trace_id": "c9b9605d5554acce9f0049f806b6b4f0", + "request_id": "189c310b-1583-4f46-a84e-33cf923c2a29" + }, + { + "utc": 1795102332.36428, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.020129374985117465, + "trace_id": "d4bff50788f0d1c0dffda28d6f926d37", + "request_id": "a2755816-5e59-4dbc-8bdb-f65889837de5" + }, + { + "utc": 1795102332.863534, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016518249991349876, + "trace_id": "4834cb7d41fd0a451bdb22932adcb2bf", + "request_id": "060171c2-6f92-4695-beb9-90d3dc698f9a" + }, + { + "utc": 1795102333.363542, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00558762502623722, + "trace_id": "26efda417e06661cd828c8e8041d1404", + "request_id": "0372c98a-861a-4f70-bac7-ce895e9ed803" + }, + { + "utc": 1795102333.36356, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011246457986999303, + "trace_id": "bbdc28a0e75ca8c0d848c258ba4f2da1", + "request_id": "035d47f9-ff92-441e-9ad3-d1e88bbd1e7d" + }, + { + "utc": 1795102333.863976, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007210125040728599, + "trace_id": "802ae21235d713413e684cbbb8c407b5", + "request_id": "89cd2091-1699-4de1-9030-cc37b3c28b7a" + }, + { + "utc": 1795102334.368006, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015296292025595903, + "trace_id": "64fdc89ab50553e3a719a02514beea8a", + "request_id": "995c59ea-6d42-43f1-93c3-31c4ec0f6109" + }, + { + "utc": 1795102334.8640242, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015549417003057897, + "trace_id": "43aea4ae0fe90480f89f07a034a54170", + "request_id": "0bcfcae7-0023-4238-b6d5-182608c5053e" + }, + { + "utc": 1795102335.366543, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01159112504683435, + "trace_id": "4d15f8b8775522632523973fec83a169", + "request_id": "94df91bf-4864-4faf-a342-cf797d1e5339" + }, + { + "utc": 1795102335.3665621, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01811245799763128, + "trace_id": "9103540d88e4313ceb032466036e4b6e", + "request_id": "a62fdbb4-0b06-44aa-9947-8b62cbde3afb" + }, + { + "utc": 1795102335.867903, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005253958981484175, + "trace_id": "72b9b16779289307838cdadf0907a074", + "request_id": "27523b22-000d-483d-9c17-48e87ea1a1e3" + }, + { + "utc": 1795102336.3636138, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016183833009563386, + "trace_id": "cfa704e87a745a1d54f7cf4f0c3bfb79", + "request_id": "95b8ede6-6cb0-4489-8584-e8b374ac83be" + }, + { + "utc": 1795102336.863769, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.012007042008917779, + "trace_id": "b3d647cd93203a8bb0786247a0a8242a", + "request_id": "1effe360-7b5e-4f15-8cc4-e7c4f5aa6038" + }, + { + "utc": 1795102337.3679411, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00913104199571535, + "trace_id": "0a3c59dd45de09b63296f0e259c815a0", + "request_id": "a6c88a2b-acbf-4cdb-a21f-754069ef9e36" + }, + { + "utc": 1795102337.367978, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.014588584017474204, + "trace_id": "31763a4b617412eefe0506e69001acdf", + "request_id": "477011e8-ef8e-4be3-8a34-f045ee866162" + }, + { + "utc": 1795102337.8679621, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007502500026021153, + "trace_id": "4d8f1afbb55262233508e87e155e10c1", + "request_id": "1898a8a1-1aac-43a4-81ad-41857d1058e4" + }, + { + "utc": 1795102338.3675818, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015203416987787932, + "trace_id": "fd414fb1be6dfe56ec5c4cf3fccb26b6", + "request_id": "9681cf06-1ff0-46ad-b6f2-7c1f11b8968d" + }, + { + "utc": 1795102338.86541, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016402625013142824, + "trace_id": "1924d0f3f2eaa457cfb511be4a076bbc", + "request_id": "73672852-ecf5-4542-80f5-565ebe2ede5d" + }, + { + "utc": 1795102339.363673, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.010883042006753385, + "trace_id": "6343a660638a14d91ed824a85e71e715", + "request_id": "41a1f9b6-8f6a-4722-9a2e-077892a6be13" + }, + { + "utc": 1795102339.363704, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.021638125006575137, + "trace_id": "fbac71a1642a287f9fe32cb17e8d1ef7", + "request_id": "e02689cb-7025-4b29-901d-26ddbeb971c3" + }, + { + "utc": 1795102339.863776, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006789207982365042, + "trace_id": "f1bfa0b943941c8192f30446289cafd5", + "request_id": "f8281709-d610-473f-95c6-e8a4a693289e" + }, + { + "utc": 1795102340.366526, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.020353958010673523, + "trace_id": "5bf29b1e5f3708eb9cb99879872f7bb4", + "request_id": "5d16cc6c-e7ca-499c-b358-d250fae0cd9b" + }, + { + "utc": 1795102340.867979, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01774637500056997, + "trace_id": "5343c864f81016923b9bdeea961d87b2", + "request_id": "f7ae1f23-c4e9-46d4-a552-ad629bd6c97f" + }, + { + "utc": 1795102341.3658738, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01992525003151968, + "trace_id": "e50ecedf9dbe0f71e55833961919ff78", + "request_id": "6ab50ba4-2100-4241-8d7e-f46c93ebdd9a" + }, + { + "utc": 1795102341.3659189, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.07042479200754315, + "trace_id": "6c2d44405b4dfd9620f6c33ad92019c6", + "request_id": "f890badd-aa98-46c5-b3ef-9db0cafa80c9" + }, + { + "utc": 1795102341.864695, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006262791983317584, + "trace_id": "ce77a660824448488874a837a17aa91f", + "request_id": "b7017045-f458-48e4-8139-bde6a8aee1dd" + }, + { + "utc": 1795102342.3636868, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01659779204055667, + "trace_id": "d87aedd0cd83eaa6e19f323530a145aa", + "request_id": "6f841d16-a1f1-448c-8fe3-3165c947613c" + }, + { + "utc": 1795102342.8679209, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.014999624982010573, + "trace_id": "1ed1e678cc239c2d43e332bf280eca86", + "request_id": "bf682b06-e5e5-4a91-8ce6-8261651623d7" + }, + { + "utc": 1795102343.36798, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017883915978018194, + "trace_id": "daf572acd39729549ddca4f47d6268f3", + "request_id": "1e3b9272-e0df-4b82-89fa-9aab15516e58" + }, + { + "utc": 1795102343.368024, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.027961666986811906, + "trace_id": "404a12ee49cb8b37e96992f339e78cd2", + "request_id": "139c8e84-9e85-4223-a668-a3d6d081f98e" + }, + { + "utc": 1795102343.867848, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005886291968636215, + "trace_id": "8b41aaa537f4ee65c380bcf8f6ae7b1e", + "request_id": "19dbd878-e53b-45a2-b003-088247adfe4e" + }, + { + "utc": 1795102344.368011, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01702020800439641, + "trace_id": "3178d460eaf6bd36d2f5fd5a759e7994", + "request_id": "0934f16a-a141-4ffa-98ff-073c9df64ab5" + }, + { + "utc": 1795102344.867977, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01055716595146805, + "trace_id": "67d31ac56d067f6c272f0827626a44e5", + "request_id": "c7604717-c9d8-4cc5-9535-2da815b84321" + }, + { + "utc": 1795102345.364053, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.020117541018407792, + "trace_id": "fb363e69eb01223ffd83a018c3ea4427", + "request_id": "420154c6-0b89-48be-a205-34eed9d4d13c" + }, + { + "utc": 1795102345.3641222, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02985450002597645, + "trace_id": "59937bc419fe8f927f84b1e5756d8cd1", + "request_id": "64524952-c30a-4409-b405-ffb66e074af5" + }, + { + "utc": 1795102345.8639271, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005657166999299079, + "trace_id": "ad180e8fb00058a4318640494540ef99", + "request_id": "b1912de9-0b97-491e-b1f0-1210f307b46a" + }, + { + "utc": 1795102346.363383, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009066667000297457, + "trace_id": "81468f57378f295ee880dfebddf17ea1", + "request_id": "769e5bf7-d544-4a6b-9f0b-488b3835b642" + }, + { + "utc": 1795102346.864492, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.014061667025089264, + "trace_id": "ee9f5d76535a44ce526a2649243947fb", + "request_id": "b9b7a0c3-ff41-467b-b31e-11c2aa1fdedb" + }, + { + "utc": 1795102347.367786, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01308941695606336, + "trace_id": "e9ab1be5f3256133b9e627ed868aeea3", + "request_id": "61b22f52-d525-44c3-9e1e-7a156a477c54" + }, + { + "utc": 1795102347.3677928, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.021234749990981072, + "trace_id": "60426fc3adc3da720e5971280eb343dd", + "request_id": "5321df55-667c-4e69-8409-4759c0f2fdeb" + }, + { + "utc": 1795102347.8678248, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005609542015008628, + "trace_id": "ab6f9616f237290c27786fc9bd1c1797", + "request_id": "3365ef14-75ee-4f51-a4b1-a912109701cd" + }, + { + "utc": 1795102348.3639412, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005726000003051013, + "trace_id": "5af8bf4a18514ef23c99c9e6f5d57282", + "request_id": "6c809b00-4e12-444a-b013-4c23ac91294c" + }, + { + "utc": 1795102348.864009, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016585375007707626, + "trace_id": "0a982ac7fd155f4b86eb6e859c49d3f3", + "request_id": "79a6edfc-5a32-4488-a257-c9e6347be610" + }, + { + "utc": 1795102349.3637388, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016458707978017628, + "trace_id": "3f29a588ff4508bf289742cddf5d2353", + "request_id": "3efffd4a-f392-4c3a-a684-cb9c022a3e22" + }, + { + "utc": 1795102349.36378, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.026289874978829175, + "trace_id": "274f7369ea389ee6e12d378a031d37f2", + "request_id": "6b52a722-b751-4765-9007-f5facac12c95" + }, + { + "utc": 1795102349.864554, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00739454198628664, + "trace_id": "1fa634dd7e59459335475ab336b1df45", + "request_id": "d8bfeaa9-e007-4b5a-8830-2c02e1a62560" + }, + { + "utc": 1795102350.3629172, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00836254097521305, + "trace_id": "40b3a20351215e9069b8b41591a08879", + "request_id": "6730b49b-230f-4dea-845a-2c40c8415b88" + }, + { + "utc": 1795102350.867844, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015740000002551824, + "trace_id": "333429fe4c94a0f96cc24cc42d0f124d", + "request_id": "ae21c329-9f65-49ed-9608-afe537dbe044" + }, + { + "utc": 1795102351.3679621, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.02002704201731831, + "trace_id": "b390f2bc67f6e2c9f3d93cdb4027efb4", + "request_id": "af560c86-4643-497d-b500-c6cd96c2ec32" + }, + { + "utc": 1795102351.368169, + "phase": "after_db_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03138945798855275, + "trace_id": "648e1290d2f2864187f12feba8664d5f", + "request_id": "6009fac9-3dfe-498b-863a-45abd42fe9ad" + }, + { + "utc": 1795102351.867893, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006248457997571677, + "trace_id": "2ab13c182033e0171b2112456977f344", + "request_id": "7515a2fa-943e-4e7a-a47c-9fe723dbb417" + }, + { + "utc": 1795102352.3648372, + "phase": "after_db_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.014936292020138353, + "trace_id": "823fdcbb132362f1142897ca95068aef", + "request_id": "8e4c0392-a66f-4d63-9623-08a07cdc7f7c" + }, + { + "utc": 1795102352.8681111, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.0071852499968372285, + "trace_id": "966f33e0f6df49af6b4e2efeedbaa133", + "request_id": "ed6de34d-9123-44fc-911c-102ec2b7de80" + }, + { + "utc": 1795102353.3638961, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017998166964389384, + "trace_id": "8626a152d8dfc6f7fee10dd8c7f89a74", + "request_id": "d1ea91af-b32c-4937-9fe5-d48d7289121d" + }, + { + "utc": 1795102353.363957, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02987166598904878, + "trace_id": "325aa1523cb1b50342e632594eabb53c", + "request_id": "7078399d-1381-487e-bd22-01a37f82ce02" + }, + { + "utc": 1795102353.865458, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017826833005528897, + "trace_id": "6d574acf93d95ad95d9215ffddb18228", + "request_id": "9c3a9ac3-049d-4f76-9fd5-c3ccd42ccfea" + }, + { + "utc": 1795102354.3679419, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01635412499308586, + "trace_id": "d8e7335e5bc23c6f813a8c10a46118a1", + "request_id": "df03c05d-542a-40bf-848e-75d145400095" + }, + { + "utc": 1795102354.8636339, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.006060791958589107, + "trace_id": "3e8687ed1d81827ad5fd5f059a7259cc", + "request_id": "63dfb348-1886-4f01-b22f-3d8d2acee295" + }, + { + "utc": 1795102355.3639271, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017612291034311056, + "trace_id": "a14a664980d82a972eae7de8535ceb86", + "request_id": "d932b887-f1b0-4b46-8906-4b91d754affc" + }, + { + "utc": 1795102355.363956, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.027799125004094094, + "trace_id": "d379d602d7d26a9a5be05627d25c4fff", + "request_id": "2f0760ca-2636-4647-b39f-aa15fe86af57" + }, + { + "utc": 1795102355.867947, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.0172491249977611, + "trace_id": "3b906a32ccf9cbf03f660e1bc5cad075", + "request_id": "21ea7e31-bf5a-4a36-b263-1d6f7dfce068" + }, + { + "utc": 1795102356.3646789, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01584533299319446, + "trace_id": "3a3f128142e798c593549265c66175d2", + "request_id": "0b9f82b9-c681-4c58-b249-38c1316bc2fa" + }, + { + "utc": 1795102356.866796, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.005968416982796043, + "trace_id": "8cf2cb64fcdd685c9f9107fb44e942ad", + "request_id": "45d5ac9c-ca80-4199-8068-7ccff91f68b0" + }, + { + "utc": 1795102357.3646579, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017876375000923872, + "trace_id": "35c2467a9da687dd010821da06e7a31c", + "request_id": "58894d96-6a29-45af-90f4-66d6741ce083" + }, + { + "utc": 1795102357.364699, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029098957951646298, + "trace_id": "2ee78b7654d7d51abede465526e2a4d0", + "request_id": "6573a6ed-6297-4d92-84f6-f1c2ebf8ae2d" + }, + { + "utc": 1795102357.863999, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.011907458014320582, + "trace_id": "673bf8c1f182a9558b403e0972ddfb79", + "request_id": "514057ac-7dc6-485c-8624-1c1d0fae63eb" + }, + { + "utc": 1795102358.3653388, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014974749996326864, + "trace_id": "7cca7db0edad816c3ec40b84fdae24ea", + "request_id": "3ffce810-4f49-4fcf-9675-86b7d12cde0c" + }, + { + "utc": 1795102358.8661618, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01681808300781995, + "trace_id": "d2007898343eced066ed730483eb90b5", + "request_id": "eb3baba2-2f12-42eb-8671-fb7b926da700" + }, + { + "utc": 1795102359.363797, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01761958299903199, + "trace_id": "eb3cae4d74eaf33625cf60e385522ab9", + "request_id": "1996152b-3532-43d6-a217-e7f8e35847bd" + }, + { + "utc": 1795102359.364346, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029489499982446432, + "trace_id": "aa828baee0935e12618a004056ae7bfa", + "request_id": "935238ab-db91-4cf8-babd-db6ff0b4b045" + }, + { + "utc": 1795102359.865116, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.011883875005878508, + "trace_id": "529fcfacfca217773957023c3076403d", + "request_id": "1d341ca0-be7b-4614-9b97-085afa2126b8" + }, + { + "utc": 1795102360.364588, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.0173005840042606, + "trace_id": "cd641318759e73c9f58ded9fdde9c014", + "request_id": "cc598939-fe0f-44d5-abeb-d01c31cea3c6" + }, + { + "utc": 1795102360.863233, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01910783298080787, + "trace_id": "a746c0188863009d5d9c4eaad7342c43", + "request_id": "fcea0705-88d1-41c7-b4dc-8f718c97a045" + }, + { + "utc": 1795102361.367908, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007552957977168262, + "trace_id": "49777ebeacce7a10f14734836c6b44d2", + "request_id": "0d746e29-ebac-4c38-8a6b-f9f9fce5b65d" + }, + { + "utc": 1795102361.3679528, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.013262749998830259, + "trace_id": "e199fb12a8d57ea187fe2381d7c73f0a", + "request_id": "a96f3be7-2e9e-4c7c-a117-283a3a703335" + }, + { + "utc": 1795102361.8679318, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.00860412500333041, + "trace_id": "7bc3fe0059ed1b37076ba69fc0243a5c", + "request_id": "2d838051-0c1d-4dc5-ac28-3bc6145b7329" + }, + { + "utc": 1795102362.363683, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017563500034157187, + "trace_id": "99705f04180b2b0df465d0668ff56493", + "request_id": "7c351b22-586a-4d2b-a504-1e421305204e" + }, + { + "utc": 1795102362.863111, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01024187501752749, + "trace_id": "b26bd8291b2be867556c80f45e854d12", + "request_id": "1ae9cba7-4a81-49a7-aa83-41b992a5c782" + }, + { + "utc": 1795102363.36783, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.00849229097366333, + "trace_id": "fe4bf6bffb47cff4323c7cfd897f9e28", + "request_id": "f7120805-6aaf-4bd7-b22e-883209c0c40b" + }, + { + "utc": 1795102363.36786, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.014617874985560775, + "trace_id": "214eef2287d1887f8a1a64decf2d9a40", + "request_id": "3d9650eb-55cf-431f-a3e2-a81e2f086f75" + }, + { + "utc": 1795102363.864008, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.009394082997459918, + "trace_id": "f18121dbda4e2395faae9657aade2825", + "request_id": "c55719fc-f668-4015-9397-74649c9683f7" + }, + { + "utc": 1795102364.367899, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.018787625012919307, + "trace_id": "9511220e74b26b5e574e044e3718aaa0", + "request_id": "c1de7d83-1243-4c22-8dcf-717cfc36e77e" + }, + { + "utc": 1795102364.863233, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007511625008191913, + "trace_id": "6087ed285def338d75038fb32882d16c", + "request_id": "f708dcc8-81bf-45de-8ca8-e6b8be1c9263" + }, + { + "utc": 1795102365.3629, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015252624987624586, + "trace_id": "7ed9d9f76469a4828a9676545edda34d", + "request_id": "82faeca5-8508-430c-94a7-44813109ae4c" + }, + { + "utc": 1795102365.362941, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028005166968796402, + "trace_id": "b57bb1ccd6c885ab907e2afeb3036518", + "request_id": "73f472de-2c5b-4aa0-838a-eb1f08b05113" + }, + { + "utc": 1795102365.8654048, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.02329904102953151, + "trace_id": "3861fc198fda248ff439c680769e5e09", + "request_id": "35a1e7d3-ba6e-4636-8e78-34b36eea0baa" + }, + { + "utc": 1795102366.36789, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.0171373329940252, + "trace_id": "f453b7858e82b6000731d078a8d58928", + "request_id": "1a707ad5-03b2-438d-803a-03154b278434" + }, + { + "utc": 1795102366.867755, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.006310040946118534, + "trace_id": "250d8e656f5ee0772f6a19d4bfcc69be", + "request_id": "91468c74-f075-4232-b52b-14df7b83415e" + }, + { + "utc": 1795102367.3679051, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.022890917025506496, + "trace_id": "fd656663fe453e92a16c6cd1266d5b6d", + "request_id": "97ed9a4e-0329-449a-aef6-ca9b17393f91" + }, + { + "utc": 1795102367.3679428, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03549916698830202, + "trace_id": "9abb3fe2f765650dceb92a3edaebd8d8", + "request_id": "b6ed1f86-9320-44ae-9d8b-2525db8e137a" + }, + { + "utc": 1795102367.8638442, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.005361457995604724, + "trace_id": "2557ca115cabe11ea03179a35b99e381", + "request_id": "d5a28cf8-ff84-4c99-9ceb-3342e95ba9f2" + }, + { + "utc": 1795102368.3639889, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.020317459013313055, + "trace_id": "b13976b24ef3fe83a18cf69b9c7d20fb", + "request_id": "c94f6188-96b7-4faf-9d14-130eb41aac2a" + }, + { + "utc": 1795102368.863532, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.019128042040392756, + "trace_id": "ce5bf2017017a9c27741730996b41ca5", + "request_id": "f5a06e47-31ae-4b2c-bdd3-458ee3e1a539" + }, + { + "utc": 1795102369.362966, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01725425000768155, + "trace_id": "4d90d99ce1dbcd7878c2fb94c641c2fe", + "request_id": "3d07c314-d551-4c16-a62f-02e61dbe8a77" + }, + { + "utc": 1795102369.3630059, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.0304619999951683, + "trace_id": "0ae846c4efeca79496ec5ffb64cc0a29", + "request_id": "9046758d-e162-49a2-a464-01a5d06e9ac7" + }, + { + "utc": 1795102369.866834, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.013736875029280782, + "trace_id": "316cbc548c33443195ef86f76a431a1c", + "request_id": "f249ab0f-9829-452b-891d-321d0dce9205" + }, + { + "utc": 1795102370.365439, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01631475001340732, + "trace_id": "db169d5dce06b03b64868a9031a8a8cc", + "request_id": "3a6dd104-11c4-4fc6-b38e-06b6385bfd7b" + }, + { + "utc": 1795102370.86714, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016819458978716284, + "trace_id": "1d550283d52eba7866d92a4aec18b6a4", + "request_id": "436087a1-26c8-453a-ad5a-3b9c450e0ce9" + }, + { + "utc": 1795102371.3640692, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.022834250004962087, + "trace_id": "efd522c412a07162b52078e83fef3dd4", + "request_id": "0b207768-7872-43c9-9b7e-446ed670fb88" + }, + { + "utc": 1795102371.364118, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.035293416993226856, + "trace_id": "ea895beddace8b12d31ef2981888cbd7", + "request_id": "dbd1e8ee-e628-433a-b8db-473368780bc7" + }, + { + "utc": 1795102371.863909, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.019783708034083247, + "trace_id": "3bc1343c4f74ecba4e046127da56571b", + "request_id": "40623e3e-d6b9-4779-8b20-a61966bbb988" + }, + { + "utc": 1795102372.367831, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014884333999361843, + "trace_id": "1ce8953fdc57eee6695d444a70789111", + "request_id": "b9b84192-3586-4d2b-9182-239b001c1834" + }, + { + "utc": 1795102372.867808, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.011282416002359241, + "trace_id": "040fe95e9be5c228e72c37acc989bcd9", + "request_id": "26db2f2b-79e0-4b4e-9272-6fd3b415ed59" + }, + { + "utc": 1795102373.363318, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017031167051754892, + "trace_id": "e66b895bd081187dc7e4647af0255d61", + "request_id": "e9c52add-96d4-42ca-b2a4-7ba62bccc967" + }, + { + "utc": 1795102373.3633962, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028464583039749414, + "trace_id": "07d0ee376ecea6d3830e5df236d1214d", + "request_id": "702fd2a3-b662-4c32-b9dd-fc70e47c1dfc" + }, + { + "utc": 1795102373.863891, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01667162502417341, + "trace_id": "0699dd9069d5f0e8c3b19f9455c3de98", + "request_id": "db66b123-d85b-4a7c-8970-520bb57e6c5f" + }, + { + "utc": 1795102374.367867, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.006138208031188697, + "trace_id": "335c2534dd93bc63ae720f031a477d40", + "request_id": "d5276239-4006-43f5-a312-3ccb53413102" + }, + { + "utc": 1795102374.866005, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.013863915984984487, + "trace_id": "f0af3b21d7b249c08d16a2575f4e034e", + "request_id": "92de8daf-e70e-4f28-b569-1f90281ba85c" + }, + { + "utc": 1795102375.365043, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.020227708970196545, + "trace_id": "34452e9a46b5296708388c9014e53349", + "request_id": "b7b9ffb1-f0a2-4a3d-b22b-c5fafcf0df68" + }, + { + "utc": 1795102375.365088, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03104324999731034, + "trace_id": "de1d9359864cf0932c42383d81b05ade", + "request_id": "1079780c-c853-4187-b800-80f2956c25fa" + }, + { + "utc": 1795102375.867868, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01748987502651289, + "trace_id": "cedb972500b0348f32687190148ca655", + "request_id": "75be9240-3ec2-4bce-833d-fd4a1a45df0c" + }, + { + "utc": 1795102376.3677812, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.006540916976518929, + "trace_id": "0d9ec9b434435c98c57f3d205bf9001b", + "request_id": "8e76d774-f4f0-47c8-8cd7-c39457883dd5" + }, + { + "utc": 1795102376.86489, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.012506416998803616, + "trace_id": "4f53f4bb9e99856b7d0e19d5c586f323", + "request_id": "e21cf087-988d-4b7d-9557-c2b70f7b3d16" + }, + { + "utc": 1795102377.367879, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017903624975588173, + "trace_id": "baab28040647c76ff9a6b798af3bc99e", + "request_id": "2bf3888c-1a7e-4cba-9d2b-2b7ad0ff444b" + }, + { + "utc": 1795102377.367923, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02961750002577901, + "trace_id": "12cc96d4381ed73e25114abe0001b995", + "request_id": "8acb714f-4511-4036-894a-c358ab680173" + }, + { + "utc": 1795102377.8652759, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01719033397966996, + "trace_id": "19a21b48031913e4a75b006f0f12d47e", + "request_id": "bc607f23-c9d8-4674-8664-84866fba0768" + }, + { + "utc": 1795102378.363302, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01607008301652968, + "trace_id": "8d6a1b4d59431dc1a98d9b94d2f5b5e7", + "request_id": "9ec4c389-edab-4e04-aed6-4249ccfef031" + }, + { + "utc": 1795102378.867871, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015584124950692058, + "trace_id": "03925141b9777799f25ef14717e4b8d1", + "request_id": "319414d7-fc41-47f9-beb8-72c4c99bbf96" + }, + { + "utc": 1795102379.3630269, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.010772958979941905, + "trace_id": "db8ebffc6c81eee6312d5f70f449f565", + "request_id": "8b57a4b8-33fb-43e6-930b-058ef407cf0a" + }, + { + "utc": 1795102379.363067, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.018079375033266842, + "trace_id": "54ed2317be4773c9614e401bbfd00a6d", + "request_id": "a61aa6a3-80dd-4158-9a26-11385a30b9ce" + }, + { + "utc": 1795102379.866941, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016292374988552183, + "trace_id": "4586536c8068ee7b2e606efbcef79c0f", + "request_id": "c2794d13-bec3-4e70-9148-33fe28b637e6" + }, + { + "utc": 1795102380.366053, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01765741704730317, + "trace_id": "7a9b0b2c89cdbe9c6a1353ae64d9c3a4", + "request_id": "e53fbabf-1bc8-42b1-98df-4e817e04e1fa" + }, + { + "utc": 1795102380.864383, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.006812583946157247, + "trace_id": "2aeb9a70eec91894c911df8d99ee7573", + "request_id": "a8d63f1d-f9bf-46e8-a539-dc3f5e6dc903" + }, + { + "utc": 1795102381.3678641, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.010398250014986843, + "trace_id": "145185be2cc62aef61b16dc256f587d1", + "request_id": "5afcf917-e947-4349-b579-6baaf63097c1" + }, + { + "utc": 1795102381.367902, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01744579098885879, + "trace_id": "06d546f52312b5ebc3fc7c4e6eada68d", + "request_id": "f4fdf684-e162-4587-a25a-a00314ec6422" + }, + { + "utc": 1795102381.865809, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01672816596692428, + "trace_id": "7be7dc0d54ec662b019b044708ad3914", + "request_id": "0138a419-ab02-47aa-8403-e80cbd6a2c98" + }, + { + "utc": 1795102382.3645031, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016298583010211587, + "trace_id": "4c24f5b724bee02c5f0f10cf4384c5bd", + "request_id": "9072be30-2795-4174-b380-b4c1d4c46d9e" + }, + { + "utc": 1795102382.863765, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.006421666999813169, + "trace_id": "16ae0d1566b71ae00ce48bb2a26862c9", + "request_id": "19422fcf-d220-4a81-a864-4dbc8219a3df" + }, + { + "utc": 1795102383.364162, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.033261208969634026, + "trace_id": "28c82085ca45bec384e656d1c63ba09d", + "request_id": "569ac268-56a0-49fe-a9eb-fb9811454bd9" + }, + { + "utc": 1795102383.364207, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.040728499996475875, + "trace_id": "2b1592ca18aac08bbd14b3f1825f0816", + "request_id": "bad9f74b-1237-475e-b7a9-13e04b9457fb" + }, + { + "utc": 1795102383.8636868, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016781667014583945, + "trace_id": "a0800dc49026f7d34353a38eb51a23fc", + "request_id": "bb8744fb-2c51-4249-8d4c-b32e61ac3072" + }, + { + "utc": 1795102384.363842, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.008170749992132187, + "trace_id": "a6db46b3a468807291b5f93d0a0bec37", + "request_id": "aaf5a793-b602-4772-9401-23d4ab93fbd9" + }, + { + "utc": 1795102384.8678021, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014613250037655234, + "trace_id": "651c5cf9b278a03c267b607f4de996bc", + "request_id": "a7783d1d-cbea-4bf6-a836-6598e50d3b31" + }, + { + "utc": 1795102385.3678, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.019336124998517334, + "trace_id": "05c7507d0c17ebdacd317620e700d257", + "request_id": "f06fc87f-555d-4466-9766-8a1192aea0a1" + }, + { + "utc": 1795102385.367832, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.027644958987366408, + "trace_id": "093f9d7d708700dd71be2f2cc0b55c13", + "request_id": "85b5a0e4-b73f-4ac9-bf39-f985acf56935" + }, + { + "utc": 1795102385.86538, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016696833015885204, + "trace_id": "3b32f4dbe81d9f37ebf1b4ad716fe121", + "request_id": "17f07dbc-b3ca-4f0c-b073-4fe0cb8443e2" + }, + { + "utc": 1795102386.3653388, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007634291017893702, + "trace_id": "9e4c1b07481604c1d490ac26107dfcb9", + "request_id": "674b446f-288e-40a7-8e00-87a8167d010b" + }, + { + "utc": 1795102386.86367, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.008551999984774739, + "trace_id": "56b46ef36cdfc348bb6e1a3397da3b26", + "request_id": "0a1e1a0c-9ab9-460b-8359-ff211904382c" + }, + { + "utc": 1795102387.365047, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.00762408395530656, + "trace_id": "ac579e3f4b380c4e02e706969961feee", + "request_id": "12353607-fe0a-442e-ab3a-7ee4db821acc" + }, + { + "utc": 1795102387.3650959, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.012304541014600545, + "trace_id": "adc673dc6646a79958b832f6e38c44b1", + "request_id": "db8e7903-cded-4bbb-b699-e1b1f61cd4e8" + }, + { + "utc": 1795102387.8648498, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016823624959215522, + "trace_id": "3c4d9c95fd103617170eb94c1549456c", + "request_id": "c9b8f184-74ca-418f-a2eb-e2c5e5dbddbf" + }, + { + "utc": 1795102388.364345, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.009671042033005506, + "trace_id": "16c58ba3cba67c606e0ddc1311d1c3f6", + "request_id": "194e3c66-78ea-4143-8821-2fe4156725cb" + }, + { + "utc": 1795102388.867729, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014467333036009222, + "trace_id": "4c1f954d4d0694eac424cf8162a96a51", + "request_id": "6bd7b5a5-301b-4e79-9c42-218b46a048d7" + }, + { + "utc": 1795102389.3677208, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.008537375018931925, + "trace_id": "dbbb3bd806fa32a78940021e423f273d", + "request_id": "45887e41-b279-4d5c-803f-8e61d147a33e" + }, + { + "utc": 1795102389.36774, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.015062124992255121, + "trace_id": "57efdaf8c960851982800243207d5b78", + "request_id": "b0296017-bb3e-4e03-805a-1f1cc95427c1" + }, + { + "utc": 1795102389.863816, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01618916599545628, + "trace_id": "cfa420fb9ff80dd1222d1a61cf2a1cfc", + "request_id": "8bf9e48e-70e6-4efc-9652-9150d92bb0ad" + }, + { + "utc": 1795102390.363383, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.019965625018812716, + "trace_id": "7968a07f77c9af31224ad0b5a0cda363", + "request_id": "694ceaf6-35cd-40ef-8ff1-8d424fac9a21" + }, + { + "utc": 1795102390.8641129, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016678583982866257, + "trace_id": "f30456c79faccc2ce9e01777b313feb9", + "request_id": "1edd4070-1d98-42ac-b0c3-d7797c6b0537" + }, + { + "utc": 1795102391.364584, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017221791960764676, + "trace_id": "fe5fef086e4fbac234e9c421c92bdf65", + "request_id": "eeaf8c96-d263-402b-a2d4-6f81085d71b0" + }, + { + "utc": 1795102391.364841, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028429416997823864, + "trace_id": "f85236b89c1b124037b1340c999d1e39", + "request_id": "3deccb1e-e6ac-4b0e-bd3f-748cf82dc845" + }, + { + "utc": 1795102391.8662488, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01131962495855987, + "trace_id": "dd8a230c95d53531f905898987c10734", + "request_id": "5a62b5cb-1624-4494-a822-6282f0715ddd" + }, + { + "utc": 1795102392.3677871, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014272874977905303, + "trace_id": "9c0ed618f957455eb70ddd130fb3619c", + "request_id": "d84dac77-1567-4c70-98d9-a6ec07006e88" + }, + { + "utc": 1795102392.8635461, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.0158273340202868, + "trace_id": "79c1318a8f882ac280aaea1571a7df3d", + "request_id": "7eeba8a3-f49f-45f2-adb7-b9df3afbae6d" + }, + { + "utc": 1795102393.3678122, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.018318291986361146, + "trace_id": "6adeaabf6a438464b22e5314ad5d94c0", + "request_id": "9125a51b-972e-4061-be09-d5151f7e20e2" + }, + { + "utc": 1795102393.36786, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030701791984029114, + "trace_id": "f72a4075d3e668b63ec83708e8d5a35e", + "request_id": "9b424849-43ba-4aea-9baa-27553043cefe" + }, + { + "utc": 1795102393.863399, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.010452083952259272, + "trace_id": "498795653085b6c8061b5a506f693a27", + "request_id": "b724fd29-969b-48f7-b995-24476f528714" + }, + { + "utc": 1795102394.365567, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.019233165949117392, + "trace_id": "077553faeb30f3b00482482d60a11e1a", + "request_id": "8ff8a0f5-b0b7-4579-8f8a-75c733a67ed6" + }, + { + "utc": 1795102394.865469, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.011639041011221707, + "trace_id": "49cd31b4d7a4d1da1fb50cae04460aea", + "request_id": "0c4c0b14-8fd6-4cb7-982c-5b65a84771da" + }, + { + "utc": 1795102395.365055, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.021357540972530842, + "trace_id": "f7dfc5d1b016529cbac912cd5875e4ee", + "request_id": "c4c45789-395a-4738-b329-fa814b8e11ae" + }, + { + "utc": 1795102395.365098, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03344550001202151, + "trace_id": "5bcc7aecb866e8f5de78dad90de2b40e", + "request_id": "d57902aa-841e-45a8-a92c-4bd7c3bee279" + }, + { + "utc": 1795102395.8636532, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.005742000008467585, + "trace_id": "b859e1afae1f72bc912d161b7ca543d0", + "request_id": "b57b412e-2e42-40a3-b4cf-0d5df4dd45b0" + }, + { + "utc": 1795102396.363779, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017466833000071347, + "trace_id": "24351dc1b2d717c285c343de0f536492", + "request_id": "4c806914-b9fc-44db-88fe-9531bc376ed4" + }, + { + "utc": 1795102396.867807, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.013305250031407923, + "trace_id": "f04e38afb753a4496f9381d6087a68ed", + "request_id": "af702626-925f-4186-8c4a-54a8b6f459d1" + }, + { + "utc": 1795102397.366952, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01584645896218717, + "trace_id": "13969cec5e84ee02751510cc74b667f9", + "request_id": "39bb2f31-eb27-4be8-ba92-9c3ef339be35" + }, + { + "utc": 1795102397.366994, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028582166007254273, + "trace_id": "bd42f9ba162dbd2408a8d3691fc572ed", + "request_id": "b1d185a1-7ab9-4d00-b122-a8a289bd2e08" + }, + { + "utc": 1795102397.867627, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007937125046737492, + "trace_id": "c111bf00dd97e6c5c485fe51bf7d593a", + "request_id": "d373450b-1dc4-40f9-9691-d805e302e51f" + }, + { + "utc": 1795102398.3671749, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.012543832999654114, + "trace_id": "c27077af5b0f739ee221583e7129f6d6", + "request_id": "0f316391-cc56-4559-af32-6029d1864fdc" + }, + { + "utc": 1795102398.8629591, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017294500023126602, + "trace_id": "e96b04b30affda8152d6c568bcd3b1ec", + "request_id": "d29d1edd-c8b0-4e73-81eb-d3049844b88a" + }, + { + "utc": 1795102399.367805, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017904332955367863, + "trace_id": "07de0726f41f240adf8db00a51b5b0d8", + "request_id": "b5d1c27d-40f2-4a17-92de-254e962af1fe" + }, + { + "utc": 1795102399.3678489, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.033103665977250785, + "trace_id": "ccd5fbc9d143090e64292ae89c0f9a04", + "request_id": "78ead772-84e3-4f7e-ac3f-284dacbad31d" + }, + { + "utc": 1795102399.863604, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014626584015786648, + "trace_id": "7f4b1aaf74e388d06865783b09b9b458", + "request_id": "a9a70e9e-9d00-43de-82be-6c447a7302e0" + }, + { + "utc": 1795102400.367747, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007335958012845367, + "trace_id": "21597d950e51b46ea3014643a78019c9", + "request_id": "16666347-d62d-4a80-b24e-14830dc51c78" + }, + { + "utc": 1795102400.867796, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017782708979211748, + "trace_id": "8114b3129a40ebe46afb783a80fb6fdc", + "request_id": "ba6b3970-2255-4fc5-ab6d-ecfaaf1afd3e" + }, + { + "utc": 1795102401.367755, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01623400003882125, + "trace_id": "04f88b84ea5479dd0d29680161f0ab44", + "request_id": "f9d94def-bd52-480c-b718-654b9607d9c6" + }, + { + "utc": 1795102401.3677871, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02844945900142193, + "trace_id": "2a04f4090089871a428465cc7793a212", + "request_id": "34f0c236-eeb4-4636-ae08-e39615089a58" + }, + { + "utc": 1795102401.864485, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016996624995954335, + "trace_id": "e8f33e53fa0a5ad67b52aeb9d063f9be", + "request_id": "0b9a7c56-d9ec-462d-ae2e-bc7dbcea3d67" + }, + { + "utc": 1795102402.3660128, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.005570582987274975, + "trace_id": "76e18d41a925d521cc5951bf75ffeca5", + "request_id": "30de16d5-8bf6-4d08-b65c-219fa4ff8e85" + }, + { + "utc": 1795102402.864426, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017862999986391515, + "trace_id": "d3ddbeddd50c3772cee0abc8a45603c7", + "request_id": "acd92447-4877-4c37-800d-d605f67ad318" + }, + { + "utc": 1795102403.365404, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01799762499285862, + "trace_id": "ef5672696dd70d0ae320f6675c4f2973", + "request_id": "2d8bc283-6762-4aaa-ba9d-4ece7da6127f" + }, + { + "utc": 1795102403.3654468, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030588499968871474, + "trace_id": "47c0e51668464bd635d883aa7eac471a", + "request_id": "0903ad76-c4a9-4aff-9a8f-19fbf91fff79" + }, + { + "utc": 1795102403.864414, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015497832966502756, + "trace_id": "72ffe912a134e7177ee2609f4eadd47b", + "request_id": "df1bde62-39f8-4a6b-9bca-d8a91e9a2843" + }, + { + "utc": 1795102404.365584, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017502875009085983, + "trace_id": "8d223ec81a8c7f078ba3080a79628e9b", + "request_id": "265a5fef-2d05-4133-a34b-081c66f1bc11" + }, + { + "utc": 1795102404.863323, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.009042541030794382, + "trace_id": "cdb7798805f5e620d9d9dc8e4fdf591b", + "request_id": "c6c5b121-aaae-485e-92b3-fa655540555b" + }, + { + "utc": 1795102405.364766, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017579999985173345, + "trace_id": "f14e3ddde5a926bed660e867810ed091", + "request_id": "b06e0b3f-4bce-4cf4-ba19-93050d603075" + }, + { + "utc": 1795102405.3648121, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029917625011876225, + "trace_id": "1ce42f20fbbb8d21b6c8cd4cdfbd10c1", + "request_id": "68befa68-8f36-43b7-93d1-81fc8639b154" + }, + { + "utc": 1795102405.8677728, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016205625026486814, + "trace_id": "0f22e568430ad0dfdce34ba9b77bf85a", + "request_id": "e564de90-1667-4732-b0ce-5fac241af880" + }, + { + "utc": 1795102406.367757, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017678040952887386, + "trace_id": "6a0e66724fb35bb62ef2ddd1b3e1f871", + "request_id": "a1c83c93-278c-420d-bef6-de4502d0ea6e" + }, + { + "utc": 1795102406.867775, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007065000012516975, + "trace_id": "1b7523f7798929c669ef7d3398955204", + "request_id": "1bd67f2c-e371-4b00-a1c9-6bd7e790ed56" + }, + { + "utc": 1795102407.3677802, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01780070801032707, + "trace_id": "0e5e2d96ab82ed1613e1c4947cac65d2", + "request_id": "25b9d1d6-449c-4884-911f-7b0a49ccd184" + }, + { + "utc": 1795102407.3678238, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03196787496563047, + "trace_id": "cc5cbf5ad8fa7be6e72432db51573e6a", + "request_id": "7521108e-1b66-4943-806c-0a787a71fbaf" + }, + { + "utc": 1795102407.863415, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016121166001539677, + "trace_id": "602293ab1ec3d9b180d3ae55952de4fb", + "request_id": "4fd2e7fb-eec9-42fb-950c-545a2d7e5261" + }, + { + "utc": 1795102408.367764, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015625083993654698, + "trace_id": "04c6cb39600d2701a179a84fbcd417af", + "request_id": "ffd3aa66-1166-49c4-8cc5-e00d5aee4417" + }, + { + "utc": 1795102408.863637, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.005561958008911461, + "trace_id": "42b05e09dced254a70c0906bed90f4db", + "request_id": "0472c0e1-0972-422b-8fae-5c0d3b9c6485" + }, + { + "utc": 1795102409.3677828, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.018918832996860147, + "trace_id": "9cfb0edc974ee2ea0d798a64b43ca624", + "request_id": "6c9897d3-d97a-4600-a14c-a61e1dc6122e" + }, + { + "utc": 1795102409.3678248, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.031359874992631376, + "trace_id": "3cf8ba4b750ff17f682d2200f0bb2560", + "request_id": "2386e8a1-eaf0-410e-b225-974dfd3c6877" + }, + { + "utc": 1795102409.863761, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015577916987240314, + "trace_id": "39c381354fc25e7be8a62af47dffaea8", + "request_id": "79de03fd-6f97-4c27-9927-456eb85673e1" + }, + { + "utc": 1795102410.367598, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01577208301750943, + "trace_id": "909b6c4b783102d7607a6d158b0ad9f0", + "request_id": "420d2205-50ec-4c2d-a9c8-40fb322acefb" + }, + { + "utc": 1795102410.867706, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.009583082981407642, + "trace_id": "6b73fd0f2b988b2813960e521938bfcc", + "request_id": "2158a0fb-98e5-4174-98d4-bd82665d7d67" + }, + { + "utc": 1795102411.3653572, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01975720899645239, + "trace_id": "7c1de695b3db8b263d5439c009ca7de3", + "request_id": "9ef35289-d204-4936-834f-863bde0bd15e" + }, + { + "utc": 1795102411.365404, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03250304196262732, + "trace_id": "aecea87159c488a39c3f7e19e132a994", + "request_id": "a6a7f219-15ec-4fd8-b998-852cba0af1b2" + }, + { + "utc": 1795102411.867764, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015792291960678995, + "trace_id": "a87c504adb499b60126ab3e1522216fd", + "request_id": "6a732e07-4d82-4e60-b6de-ad33e6e5654a" + }, + { + "utc": 1795102412.363724, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.009664791985414922, + "trace_id": "1021b97211f811a7d3964874f777da9b", + "request_id": "9d479c1f-d1e7-41d0-ba6a-220198f22332" + }, + { + "utc": 1795102412.863917, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007974457985255867, + "trace_id": "36236c8b36b9958fffb6cb529f191fe3", + "request_id": "fc49cf81-0b41-401b-9587-1323040a872f" + }, + { + "utc": 1795102413.363141, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.009677833993919194, + "trace_id": "043253dd142344be9aa1bf8d92c228b3", + "request_id": "b4d2fc36-6cc7-47ca-9b14-2e31245f1d2f" + }, + { + "utc": 1795102413.363202, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.017140707990620285, + "trace_id": "125090217568f1e95f5cf8f0fa403c80", + "request_id": "42ae3a39-066b-43d4-8b56-acea25a4ddf9" + }, + { + "utc": 1795102413.86336, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016220875026192516, + "trace_id": "a783851c8cb6320d40877510da257cec", + "request_id": "b5efa4d6-e8c0-4c94-9d50-6fdb5e60be81" + }, + { + "utc": 1795102414.363072, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016290874977130443, + "trace_id": "19683d1cea76d5f17a27806d261f2fd3", + "request_id": "89a63956-7fd0-4ffb-aac8-95c5038129e4" + }, + { + "utc": 1795102414.86372, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.012358207954093814, + "trace_id": "75025fccc36e5e3971ccb7cc4e879caf", + "request_id": "255b7304-e8b9-412b-b8d9-e1dbaf0780d0" + }, + { + "utc": 1795102415.367646, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.008794167020823807, + "trace_id": "38149f47fd00bde56518cc05ea757764", + "request_id": "83a50993-c87d-48d0-8d9a-c7bb2c79e8cb" + }, + { + "utc": 1795102415.367677, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01500741601921618, + "trace_id": "a6a9fc26d97f53e039ac8728f836c9c2", + "request_id": "430644d1-0d6b-49ce-90f4-55b9a0455a25" + }, + { + "utc": 1795102415.863302, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01248008297989145, + "trace_id": "54ece4633e937a6c5f4bc47a6615acd0", + "request_id": "2fe17669-e487-4e34-b0bf-96531ded9ec6" + }, + { + "utc": 1795102416.3677068, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.02055683295475319, + "trace_id": "accc9978da37b4d7a276e297cfd23390", + "request_id": "d2af6677-43da-4587-9b61-20e07e149564" + }, + { + "utc": 1795102416.8647199, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.00998433300992474, + "trace_id": "27fd83077407901f1ee304b584a2413c", + "request_id": "d81ccf3a-452c-433e-95ed-51fd56785db9" + }, + { + "utc": 1795102417.365332, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.018538791046012193, + "trace_id": "b618ce2e2d32e9841c913aeb19ead733", + "request_id": "aaee6bf4-4e39-47e7-9f35-b9ec8259da24" + }, + { + "utc": 1795102417.36536, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.033530417014844716, + "trace_id": "961f3cf5b47a9a4064cacd4815d0b05c", + "request_id": "edfb4976-03b1-4c4d-887d-dcc06dc22e31" + }, + { + "utc": 1795102417.8676841, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015877375029958785, + "trace_id": "64fb03abb02288ba03d696e0975eb740", + "request_id": "c8619efa-2af3-4ade-a60e-866f76985669" + }, + { + "utc": 1795102418.36336, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01580237498274073, + "trace_id": "c94115ee8ae6fe259d57a43f2ffa44d1", + "request_id": "fb5c704a-015d-4625-baa4-9ea7e87ad852" + }, + { + "utc": 1795102418.863721, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015530208009295166, + "trace_id": "5d4b8d6ba5de43ee211b45ff41b29b1e", + "request_id": "ade18107-9931-42d5-b510-6a19b0e2e571" + }, + { + "utc": 1795102419.365393, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015622958017047495, + "trace_id": "f00e00ec96ecb6205cac1985c164738a", + "request_id": "d6c82cca-2aaf-4992-aaab-db3055b1d6b1" + }, + { + "utc": 1795102419.365421, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.026343958044890314, + "trace_id": "64e6917c36017941d68b6e7c7f9c16dd", + "request_id": "02f51b01-5771-4722-bf3d-315618b7aa47" + }, + { + "utc": 1795102419.867722, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01030058303149417, + "trace_id": "68e565faf043811e28a76f711645a6d6", + "request_id": "19e5d81b-59dc-4392-aecc-fa86956aa6de" + }, + { + "utc": 1795102420.367711, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015316749981138855, + "trace_id": "897a9c6a6ea288601f004a49244adaf0", + "request_id": "5accc224-99ea-4650-b17a-a74b01c15138" + }, + { + "utc": 1795102420.867711, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01670500001637265, + "trace_id": "9c2d1633414fe925ef74fd269b58479c", + "request_id": "9e7832f1-ecfa-4536-bc64-7789c374ca29" + }, + { + "utc": 1795102421.36775, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016369208984542638, + "trace_id": "a836ee9bb315e68046d192df22aa954d", + "request_id": "c82cd4fb-fcfc-4cd6-a488-fae80ec2ce38" + }, + { + "utc": 1795102421.367799, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028319917037151754, + "trace_id": "bf2f1fdfdbe9f20b9ab53964d11d51bd", + "request_id": "9db43485-86d1-4f79-a459-30748abb6e2f" + }, + { + "utc": 1795102421.867598, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.005136416002642363, + "trace_id": "74616250570e8d881eb48ad56a1ed91a", + "request_id": "a8a640be-61ec-473a-87c3-9dba55cdfb22" + }, + { + "utc": 1795102422.367701, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016418666986282915, + "trace_id": "7d79579feb732ac58aeb5d918097fd8e", + "request_id": "3d11f52f-e04b-4eeb-831f-6428d3bcd05a" + }, + { + "utc": 1795102422.8644118, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017578792001586407, + "trace_id": "db20b52913d7cc37e06cc3b86ae7de72", + "request_id": "3259ac41-437c-4748-81a3-e4a85a8f2c0b" + }, + { + "utc": 1795102423.36354, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.02166920801391825, + "trace_id": "3fbd524a2c86fecac2cf72fcbc8b07d9", + "request_id": "197155cb-1636-4d70-a885-2d414486c7dc" + }, + { + "utc": 1795102423.363615, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03145316703012213, + "trace_id": "fc01aab31130e3cc8ae548e7e88c6e6e", + "request_id": "9d7f9060-2f04-46d6-af12-49df53c0b45d" + }, + { + "utc": 1795102423.863683, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.011149332975037396, + "trace_id": "37f864759de0a0caa81b28f33e0ba756", + "request_id": "204626a8-99c9-47b2-b730-5a2822b626fe" + }, + { + "utc": 1795102424.364724, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016318959009367973, + "trace_id": "25a43881ac07f057ecb19fb9b2935ba9", + "request_id": "08e8c33c-f576-42c8-b4f1-445671dcd2c0" + }, + { + "utc": 1795102424.8674712, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.010270832979585975, + "trace_id": "afee8ded253e1715e41c29aa9434002d", + "request_id": "327f2e9f-9533-4941-a7ee-de8890f414d7" + }, + { + "utc": 1795102425.363681, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017710209009237587, + "trace_id": "5ed2ffe9989d718781bd2021dfda3aee", + "request_id": "0db1e7ca-d8cc-4b93-9501-611835b22044" + }, + { + "utc": 1795102425.3637278, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029055624967440963, + "trace_id": "78231631555a8a92ef292fe80e9ef888", + "request_id": "4c7725cc-e49d-4125-9f93-c872bc699459" + }, + { + "utc": 1795102425.863774, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015548915951512754, + "trace_id": "24e6e6e8d53e159bdf9fd450bc3346bb", + "request_id": "f04ad3c7-ce9b-47a1-b9c8-982ea4a4e54d" + }, + { + "utc": 1795102426.364598, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.008875458966940641, + "trace_id": "8eb2b1d69706a49be994156726b9051c", + "request_id": "dae22f7c-0806-4594-9f5f-cdf50780a82b" + }, + { + "utc": 1795102426.867625, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.008479500014800578, + "trace_id": "b7ffbf4067bd55bf00b203e29f50cc33", + "request_id": "f2e800b6-5c7e-4220-933f-e43b0aca9557" + }, + { + "utc": 1795102427.367722, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.029423583997413516, + "trace_id": "bcbf0f5b1fdab32a792745131a21faa7", + "request_id": "abab8b8d-2925-4aee-a49b-2a2acf3b0c3b" + }, + { + "utc": 1795102427.367767, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03378054202767089, + "trace_id": "dabb919f158274f7c3c0cb5b1bb1aedb", + "request_id": "80213796-05e3-4e12-8e36-a2a0fbe3004a" + }, + { + "utc": 1795102427.8645172, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014543290948495269, + "trace_id": "54db78a125bc7235d918835f3b6586be", + "request_id": "b5340fcb-2206-40be-93a8-832f009927a7" + }, + { + "utc": 1795102428.364377, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.00569808401633054, + "trace_id": "a5f1fe104c750dd82e8d58bd17f201b7", + "request_id": "c4f92551-28a8-4968-8269-0ff795ee5ae5" + }, + { + "utc": 1795102428.867705, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007685457996558398, + "trace_id": "73677bec6386f585d5db1af30dd9db9c", + "request_id": "d4bd4a53-e6e9-444b-880a-2828ae22ece0" + }, + { + "utc": 1795102429.3655849, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017573583987541497, + "trace_id": "04ffe7be04a28031e08eaa39efc98792", + "request_id": "2a4a6ac2-2c79-495a-b2f0-e07b40fe9b99" + }, + { + "utc": 1795102429.36563, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029355333012063056, + "trace_id": "91a03ad85986df7c21a82abdfde7863a", + "request_id": "e4e305dc-b312-419d-8ad5-c3b14914a8f9" + }, + { + "utc": 1795102429.8634722, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014020917005836964, + "trace_id": "82f0f0da450e2dc5536a3cbc237e3603", + "request_id": "405d4154-02d4-4f59-b3f5-e2e83fa73c28" + }, + { + "utc": 1795102430.3632991, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.008760333992540836, + "trace_id": "76a0a88b1940114d68838855536f3788", + "request_id": "dc6d4fa6-65fd-44a4-8f9e-f013cbabf50e" + }, + { + "utc": 1795102430.863607, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014520208002068102, + "trace_id": "8bbe616a69adce412c2ba7da98102b8a", + "request_id": "63db6a82-b86d-435d-a268-478115465fdc" + }, + { + "utc": 1795102431.367687, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.019691542023792863, + "trace_id": "3e8b6d957d6494db9672044a3530dc92", + "request_id": "95edc82d-fa25-468b-9eac-e8f64f61e316" + }, + { + "utc": 1795102431.367732, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03015966701786965, + "trace_id": "8cffb500136afd88df8882ec0696a3d1", + "request_id": "3e285b4c-8241-4378-b3a1-46a1e759c224" + }, + { + "utc": 1795102431.8676102, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.011735707987099886, + "trace_id": "4a9be09b89ab9c23976d4ab535841af1", + "request_id": "bb1c1242-8dd1-4da6-bce8-4839dd2657c3" + }, + { + "utc": 1795102432.367671, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016700041014701128, + "trace_id": "511ef15321a069c7c385990a57e505b6", + "request_id": "f06e5686-e6c5-4ae3-9b89-7535bd56eb98" + }, + { + "utc": 1795102432.8664262, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.013757165987044573, + "trace_id": "64f1269f515a8a69f5606d6443003bec", + "request_id": "87c78b8f-34af-4c55-8260-15c0e30345a6" + }, + { + "utc": 1795102433.36763, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01437429100042209, + "trace_id": "b10be54343c856277b6378a0c96e40ec", + "request_id": "9fad8f88-0cc3-40da-a294-c53b9915f483" + }, + { + "utc": 1795102433.3676548, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.021486249985173345, + "trace_id": "af7cf74526842f886204b5dc38337665", + "request_id": "59fdba54-9234-40e6-91e1-580b63e79696" + }, + { + "utc": 1795102433.867693, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01707470801193267, + "trace_id": "04e03e7b950109cad0681884ddb867f1", + "request_id": "0150fde1-1c91-4aca-a657-6a0f4ab592a9" + }, + { + "utc": 1795102434.363307, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01608795899664983, + "trace_id": "f977fd2e46e1d4c936fdbda598bcc048", + "request_id": "24061d92-1fad-4ac1-94bd-06568f3af87e" + }, + { + "utc": 1795102434.863598, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007224457978736609, + "trace_id": "11e6ddb4d32b6544cb2ac902c3f03f41", + "request_id": "e2822e78-bd89-4c5b-993a-840064a4ef52" + }, + { + "utc": 1795102435.364865, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.011044415994547307, + "trace_id": "1278a486f67bf2a984c377acf4dac972", + "request_id": "6ef29e6f-1196-4492-a7f5-775b11279390" + }, + { + "utc": 1795102435.364896, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.016963417001534253, + "trace_id": "d1b305226c0e9cf63a1889bb86669526", + "request_id": "e7509d4c-31fa-46e3-94ee-3cdb169ab9d5" + }, + { + "utc": 1795102435.863466, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015675541013479233, + "trace_id": "f4c0446c50690020c50c76b62183a765", + "request_id": "a12d7fb6-c2d6-4a71-959b-817f19fdb8fe" + }, + { + "utc": 1795102436.3634539, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017218165972735733, + "trace_id": "175b4296b88b51ca160c5f8f447377a0", + "request_id": "d9766887-8d9f-47f5-95be-80557638211d" + }, + { + "utc": 1795102436.863505, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.0060320000047795475, + "trace_id": "f639a10f2886bf7ee9289cf6dc8b1cab", + "request_id": "8d7f1b86-38a3-4756-a995-58a887896307" + }, + { + "utc": 1795102437.363026, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.009834750031586736, + "trace_id": "d851ed4f39017923b5da1cad8d78e12b", + "request_id": "657e1229-02bd-4aef-a417-3fdd55a5cedc" + }, + { + "utc": 1795102437.363044, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.015458667010534555, + "trace_id": "3e628078050d6ee1d91e8ca20757f46b", + "request_id": "86c114c0-421d-49a5-9a78-b3e1f210744a" + }, + { + "utc": 1795102437.867669, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01662633300293237, + "trace_id": "de25d47ca3a52e2077584838fed379c6", + "request_id": "e72a5d4e-3ff4-4306-aaf5-41c8195b8bad" + }, + { + "utc": 1795102438.367658, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01591783296316862, + "trace_id": "d1001c4d6a0337569489f22b81e2ba87", + "request_id": "d377b9f9-13ee-4676-97e0-0d6e009fef2b" + }, + { + "utc": 1795102438.86767, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016395874961744994, + "trace_id": "ebd624f97acc62c13dae5c581a12757b", + "request_id": "44711340-0ad2-44fc-be30-dfa3c1bfc5cd" + }, + { + "utc": 1795102439.36733, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.02565637498628348, + "trace_id": "9a64217493529803d236fd16ddac5660", + "request_id": "8b1da0c7-88e3-4a51-9cb6-03dae3548adf" + }, + { + "utc": 1795102439.3673759, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03703787497943267, + "trace_id": "252de90f6cc3042741a354f623526b70", + "request_id": "33a79bc4-800c-4c4d-bfcd-343bf92b982d" + }, + { + "utc": 1795102439.863589, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016927208984270692, + "trace_id": "ca0010e9415b0289bb448c55d3e4050f", + "request_id": "82742ac4-cbfa-4909-a965-42b54d11254d" + }, + { + "utc": 1795102440.365728, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015344124985858798, + "trace_id": "4c53f043bec7a13c3278947ce731562e", + "request_id": "aad4ca81-c512-4ba8-91fc-6a51aff043ef" + }, + { + "utc": 1795102440.8676548, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015243542031385005, + "trace_id": "d3793bf69548af727aea91c0be5bb426", + "request_id": "35306bb4-4254-460c-9dba-475787c84594" + }, + { + "utc": 1795102441.363359, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.010325375013053417, + "trace_id": "eceb4be3c050992dd94653e9b3c57643", + "request_id": "b7a1fe48-3acf-4540-ad97-044131cc14c4" + }, + { + "utc": 1795102441.363796, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01592224999330938, + "trace_id": "74d9592aad2130320fd659e38c02e25e", + "request_id": "aad2b11c-decb-45d9-96f6-cb8e1403411c" + }, + { + "utc": 1795102441.8664372, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01578949997201562, + "trace_id": "f3ddeaacde59b3c192fa170c279fa400", + "request_id": "0295e39c-c19d-434c-9700-ac606d01b6d1" + }, + { + "utc": 1795102442.363565, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017722541990224272, + "trace_id": "00a27abf1e17e7ab358974d3abba67ed", + "request_id": "492e516f-6534-4cb3-b52b-0e363bcbc69b" + }, + { + "utc": 1795102442.8634062, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017071790993213654, + "trace_id": "517cef31bcd8c11a6f818d68b294ec8e", + "request_id": "61da7910-852f-44d7-aa45-a6d986be5608" + }, + { + "utc": 1795102443.363497, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007410165970213711, + "trace_id": "eebfb24a292eca7e86bbde718d445db9", + "request_id": "083fb47b-1669-4519-9502-fdad0841e935" + }, + { + "utc": 1795102443.3635368, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.013141040981281549, + "trace_id": "728c5c07712cb545aefdb5d996922cbc", + "request_id": "e25d3095-3606-43aa-bdc6-f0fad7f001c8" + }, + { + "utc": 1795102443.8676438, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015743041993118823, + "trace_id": "7514b26cecdba0a2984f57ab3a752099", + "request_id": "75842dff-f671-448c-9574-e68cf42ce67f" + }, + { + "utc": 1795102444.36359, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.00878520798869431, + "trace_id": "f826aefefa3e14c534462a47393efd8c", + "request_id": "069d4467-202a-4745-95ba-9e56089ed44a" + }, + { + "utc": 1795102444.864238, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.012207375024445355, + "trace_id": "b0ff26a66a1b6fc1c1fe8858064d11c9", + "request_id": "fb73cd53-3649-4297-a3c9-19fc2504b0e2" + }, + { + "utc": 1795102445.363557, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.008502374985255301, + "trace_id": "9c5a3f90e3c2b47d620076147f6647cb", + "request_id": "608dab50-62d1-4553-9d89-7237e8678ab5" + }, + { + "utc": 1795102445.363605, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01618049998069182, + "trace_id": "ca43efbd1a23bf3f94097b8220f87865", + "request_id": "bb68bbcf-d15c-4285-8c7a-5edecd7a0a58" + }, + { + "utc": 1795102445.867569, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016005749988835305, + "trace_id": "6bd71f1b16b035ae5cd474d23d5860b9", + "request_id": "74e09d32-a2e5-4ee3-b417-c3c9548fc52d" + }, + { + "utc": 1795102446.3676019, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015444999968167394, + "trace_id": "82b2b2ba66edb2ffdddfee6b131bef87", + "request_id": "dde3febc-9219-4198-882a-ed92e87b6fd4" + }, + { + "utc": 1795102446.866642, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.009087332990020514, + "trace_id": "f5f9a5de07943b4dd18e0aa36444a64c", + "request_id": "3e1ec629-08c4-4fda-b426-0e7b3b973d42" + }, + { + "utc": 1795102447.3628411, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.011773957987315953, + "trace_id": "9fbaac7ffd3a7096bf404b919977c65e", + "request_id": "a3db3856-ad5c-42c2-85ff-3ca7f3a9c39e" + }, + { + "utc": 1795102447.3628852, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.020113458042033017, + "trace_id": "698174c345241bbae298629a129705be", + "request_id": "a03bf06f-24d5-41c2-a4d5-5e6195dd4cd4" + }, + { + "utc": 1795102447.8646421, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014318041037768126, + "trace_id": "d90420037be92d93c142ee772b82e5b1", + "request_id": "23b77d39-526e-4cec-b2fe-b8c66b39ca6f" + }, + { + "utc": 1795102448.364255, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01648445799946785, + "trace_id": "c95a105ca94084c49dd239a847b75d95", + "request_id": "4b9eb19f-d361-43ab-b8da-9f92254ef1d8" + }, + { + "utc": 1795102448.863559, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01859537500422448, + "trace_id": "9da78c88731d056c5b79cd4e161f8c04", + "request_id": "7939574a-859b-4e8f-b399-da53c8646716" + }, + { + "utc": 1795102449.362666, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01855941698886454, + "trace_id": "e0cb7e8affeb8df504219e6bbbe86298", + "request_id": "8aafd884-6296-4e6e-a00e-481ca5c56d97" + }, + { + "utc": 1795102449.362692, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.027891249977983534, + "trace_id": "b0008e8e8d97c33dbba0caac2e3e2999", + "request_id": "e5476ea6-0322-4c65-91b8-8345964a8970" + }, + { + "utc": 1795102449.86764, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.006324374990072101, + "trace_id": "4c7576268a58fdc3b4a65fb98ca0c229", + "request_id": "adfe2386-98c4-4683-b15b-d0941d793099" + }, + { + "utc": 1795102450.366602, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014055666979402304, + "trace_id": "47f00ac64b76f18201f820c9f870987d", + "request_id": "cc0623d4-6ab9-4e7e-96ca-1e66c4deb3bd" + }, + { + "utc": 1795102450.867615, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016802290978375822, + "trace_id": "ff3f490c10759c06d8af46ae8cb028a5", + "request_id": "e8be0534-da44-4f6b-bc3e-8ae3ee69201a" + }, + { + "utc": 1795102451.367626, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.02218370803166181, + "trace_id": "1ae0910779c50e6bb640fc73d005fd02", + "request_id": "9e41173b-06fd-46cc-9482-38b952fb588d" + }, + { + "utc": 1795102451.367672, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.032944792008493096, + "trace_id": "bd0977d510155808da02912c4f110334", + "request_id": "5497adea-b6f0-4d52-b703-bc9d15437af6" + }, + { + "utc": 1795102451.8675358, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.0065764159662649035, + "trace_id": "29b678c1d5a79847f6496bdd6e92c520", + "request_id": "ff5c4449-d113-4664-b898-d5c376c99cf7" + }, + { + "utc": 1795102452.364527, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01660933397943154, + "trace_id": "4e9a3add802d945bfd3e3e00bb051c48", + "request_id": "ba3d0241-dd6e-4d91-be85-aa112cea9563" + }, + { + "utc": 1795102452.8676102, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016413250006735325, + "trace_id": "ddb67d11fe96eed9886017093c87d806", + "request_id": "97bc74ce-d4d8-477d-9b83-d78314939c55" + }, + { + "utc": 1795102453.363491, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014762875041924417, + "trace_id": "0681c332ba5e02319c325402b42a49ea", + "request_id": "0d685cf2-7f0f-46ef-b690-8fd1bda7dd5a" + }, + { + "utc": 1795102453.363517, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.024424750008620322, + "trace_id": "d908cf53d57705dea7dd03044d842395", + "request_id": "6acd21c0-eef3-49b3-9349-2b69ddeff87d" + }, + { + "utc": 1795102453.867581, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.012784749967977405, + "trace_id": "9897ad701adc11eca48026d9877c8d59", + "request_id": "af0f96d6-7974-441b-84e6-2ec583adf3f8" + }, + { + "utc": 1795102454.365363, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015164166979957372, + "trace_id": "89a21a0954ea808df4341c8b1b2c8712", + "request_id": "f9c3754a-5499-49dd-a7b7-ca726c99def3" + }, + { + "utc": 1795102454.8662112, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.013387667015194893, + "trace_id": "595426f07aa3fff1387337bf628b6ce7", + "request_id": "ab0a7988-21f6-48cb-9c48-bf718dfa42f4" + }, + { + "utc": 1795102455.3629818, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.019770042039453983, + "trace_id": "578f9413e73944095326e065ac2db92b", + "request_id": "22565020-d869-4cbd-9a51-46f4deaa1071" + }, + { + "utc": 1795102455.363034, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03127700003096834, + "trace_id": "8c0018380567c71260de710dddfd0b42", + "request_id": "aab08831-abe0-4e32-b464-771d15afd5c9" + }, + { + "utc": 1795102455.8635871, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016804707993287593, + "trace_id": "8095888891feb84a1091df7137b38ead", + "request_id": "8c0bd93e-3ece-4f44-b0aa-a9aa8b397c1c" + }, + { + "utc": 1795102456.3676102, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007425500021781772, + "trace_id": "539cc0bfef69b37b4c9d1a8dc51efe2f", + "request_id": "c1cf7a22-02db-4cc5-83e1-e996ad545444" + }, + { + "utc": 1795102456.867615, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.010429084009956568, + "trace_id": "eb1bf0b82c96c189ff7fd75984b6d8a9", + "request_id": "5c079102-8623-425e-8cd7-450400073005" + }, + { + "utc": 1795102457.3639748, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017744750017300248, + "trace_id": "1ff7d287715b71571bfdc4e8fd49505f", + "request_id": "a015366c-207f-42cb-b5fd-9bd1c5880d8b" + }, + { + "utc": 1795102457.364018, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028241249965503812, + "trace_id": "9f20507b1373a74d69e7effff81e4e1a", + "request_id": "29d21a90-a114-44c7-a809-ab99c20baf6b" + }, + { + "utc": 1795102457.867581, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014599415997508913, + "trace_id": "a401e931a41071a25ec3c7fe0c52da27", + "request_id": "7d5b4343-6b2a-478f-b242-89e8112a54f7" + }, + { + "utc": 1795102458.367444, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.0066549169714562595, + "trace_id": "ef545bea37ba501c2213800856d54986", + "request_id": "79027daf-94d7-4f83-8412-22be2f13fdf5" + }, + { + "utc": 1795102458.865005, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01703866699244827, + "trace_id": "3bc8a1d6f9304adf4dd18975bf2b5179", + "request_id": "e641a5b8-aa7b-4862-97c4-aff81b3e04ed" + }, + { + "utc": 1795102459.367558, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.019166083016898483, + "trace_id": "000125e6047d693bd3e8891834ed49e1", + "request_id": "f2b52dec-41d8-44ac-8ca4-0a1d21adf83c" + }, + { + "utc": 1795102459.3676019, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030556249956134707, + "trace_id": "7a327b2469aa00d3385296ccb23273b1", + "request_id": "da350f1d-7ed7-4997-adc1-92c136555630" + }, + { + "utc": 1795102459.867604, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.00954154203645885, + "trace_id": "7d77b702d83355d7c10d5a4210fce586", + "request_id": "ae15f554-9d6c-464b-b4d2-9594fac3eee5" + }, + { + "utc": 1795102460.363765, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017119499971158803, + "trace_id": "e7eeef9ee0e6ba6d26906c7d1bf72439", + "request_id": "73771b12-d729-4609-824c-e27d72ebf44d" + }, + { + "utc": 1795102460.863709, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015094625006895512, + "trace_id": "87ad5f6119f4a517ed2de3e83c36a2c3", + "request_id": "9f9f995c-1d85-43db-b898-43fd3f67de8c" + }, + { + "utc": 1795102461.364383, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.020289083011448383, + "trace_id": "f97f7b8dc9a8cd334ed10918a880b78e", + "request_id": "60069c14-9344-43ec-8270-4b93df0da31a" + }, + { + "utc": 1795102461.364426, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029962291999254376, + "trace_id": "03ad4342dfc459c7adf064cbda40f4dd", + "request_id": "6c938087-392b-4052-a9fa-699ed915ce75" + }, + { + "utc": 1795102461.867611, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.00864166603423655, + "trace_id": "b94f94f4fc6d24b6321199d7d68d6849", + "request_id": "124802ac-8313-4f4a-b81c-6510f04a53e4" + }, + { + "utc": 1795102462.3675818, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.017710374959278852, + "trace_id": "4b7fac0a4f3036769876de011204893b", + "request_id": "52b7e5d5-3a48-4abc-96cd-f5a311d5f391" + }, + { + "utc": 1795102462.867521, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.008550958998966962, + "trace_id": "54efa4c6a78835cc4ff39665d7915c38", + "request_id": "3e221912-3182-4469-959c-06b09d9b6288" + }, + { + "utc": 1795102463.3643572, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016002500022295862, + "trace_id": "3fd3a1de29bf77c31d656f1876e003df", + "request_id": "86ea1c7c-0c31-4eb6-85f1-41124ed2f9db" + }, + { + "utc": 1795102463.364393, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.023984416970051825, + "trace_id": "c298776681339dbf0f36a2895b2cfdc6", + "request_id": "9b04ca0d-d6c6-450b-9853-629669abaf3b" + }, + { + "utc": 1795102463.867564, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007661124982405454, + "trace_id": "f2d2140b2d1ec3142adbf8ad00c80e76", + "request_id": "e0dee199-8d29-4a4b-be9d-49aa032c820b" + }, + { + "utc": 1795102464.363488, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016568916966207325, + "trace_id": "7102975676f9c5adda4410b3083a502a", + "request_id": "55c833ad-7795-4939-8c7f-58250c9430a2" + }, + { + "utc": 1795102464.867487, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.0063234170083887875, + "trace_id": "0d899929fe226fabb0b8f9a0dad0e0b9", + "request_id": "682b9cae-2387-4c01-ad2a-d36307aefa9e" + }, + { + "utc": 1795102465.363529, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016678209009114653, + "trace_id": "004d6b1827ba0dda8c1de8d99841306e", + "request_id": "dd661bbc-e661-4d2a-a4c7-1fe8e4283142" + }, + { + "utc": 1795102465.36357, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02568337501725182, + "trace_id": "13fabfaed2cc759b5a3355b9f54e02ef", + "request_id": "46243096-a85e-4c0f-8862-400b28f964de" + }, + { + "utc": 1795102465.8632011, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.011167499993462116, + "trace_id": "5a9d9a874885b440d5408adaee9a28eb", + "request_id": "b72e1e00-551b-450e-813d-c406c77bea6d" + }, + { + "utc": 1795102466.363478, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015470042009837925, + "trace_id": "931e004ec4e76f7bfd618f448115085f", + "request_id": "f25f34c2-bafe-4d9d-80f8-f7f3028dc8c7" + }, + { + "utc": 1795102466.863482, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.011231042037252337, + "trace_id": "4f3ab8c02be3311eb1fb13e132f5bf79", + "request_id": "b0f1c868-629e-4b2e-a110-96b3a46f18c8" + }, + { + "utc": 1795102467.365616, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01846950000617653, + "trace_id": "c958c465d137036c8e832495c8694760", + "request_id": "bffcb415-c2f2-4d79-a96d-a8971c1394eb" + }, + { + "utc": 1795102467.36566, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030357750016264617, + "trace_id": "ef2c2736f31f27fbced7dbeaa97cdb77", + "request_id": "f332406c-55f2-42f7-af09-83c5512ab258" + }, + { + "utc": 1795102467.867069, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.028153790975920856, + "trace_id": "a14723903797f77996c4a06dcb68f679", + "request_id": "c5279080-ee13-4234-8096-4ea26999c006" + }, + { + "utc": 1795102468.3637629, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015079875010997057, + "trace_id": "5764d6f2ae8cbf751b2e88e48bdda261", + "request_id": "583dc93e-f8da-411c-987d-acdbd33c1ba7" + }, + { + "utc": 1795102468.863251, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.01642200001515448, + "trace_id": "7a9dc429b096eb7d83b8140df3565123", + "request_id": "6f47d21e-a7a2-4ff1-8097-c95148624fef" + }, + { + "utc": 1795102469.3645508, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.009073499997612089, + "trace_id": "5d679a059c07f050e58fb787406c25ae", + "request_id": "897985fb-eeff-48ab-aaf8-1f5aac80c340" + }, + { + "utc": 1795102469.3645902, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01567558298120275, + "trace_id": "160751c434f0e802f045d94788ce5e3c", + "request_id": "00d56375-7b33-4aeb-a854-d6e6cae9b6a9" + }, + { + "utc": 1795102469.86348, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015058834047522396, + "trace_id": "b220bc940997348f4f6cd8e4539e1c18", + "request_id": "0f2740d0-fde3-4b0b-b909-8de8356b2a98" + }, + { + "utc": 1795102470.367556, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015819665975868702, + "trace_id": "795e7c93d02e70121785f89787d757e0", + "request_id": "602db9cd-6d45-4a64-b7d6-4e51c05a7c70" + }, + { + "utc": 1795102470.8644822, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.014986374997533858, + "trace_id": "ec0d976547114614e60676beae5c2c94", + "request_id": "77043f4d-1119-4776-a6b3-c4076d3f3243" + }, + { + "utc": 1795102471.367477, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.007079833012539893, + "trace_id": "455f9b4a077de6bf581192a6478e280b", + "request_id": "85349b82-3574-4cb1-b9ef-d7089f2f4e39" + }, + { + "utc": 1795102471.367508, + "phase": "worker_stopped", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011628791980911046, + "trace_id": "26b664883f5fedd655bce47cefc39d32", + "request_id": "f3d7b9d3-5dbb-4153-bb20-85ddcb995c2d" + }, + { + "utc": 1795102471.8632162, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.016668500029481947, + "trace_id": "d9d1ee5cbc45220bd52dcb80d6c18bf9", + "request_id": "395204ca-94b0-4924-8ecc-ec6cd78fb929" + }, + { + "utc": 1795102472.365772, + "phase": "worker_stopped", + "kind": "message", + "status": 201, + "duration_seconds": 0.015566083020530641, + "trace_id": "9ebe9d2a7f4716469785f039785fa8db", + "request_id": "1803020c-b2d2-4f95-99ef-f503ccf5668b" + }, + { + "utc": 1795102472.8678641, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016977916995529085, + "trace_id": "2b3fd1349030cecef9ac40c45c19a6de", + "request_id": "a41d0882-438c-4023-8276-f8b54ffbfe51" + }, + { + "utc": 1795102473.362782, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0063117920071817935, + "trace_id": "c50f1e4c2ca9a3f19d480be349793df2", + "request_id": "ffa9389a-4a02-43d1-82af-104373117e60" + }, + { + "utc": 1795102473.362827, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.010864999960176647, + "trace_id": "71b4f4ded844923e7b35939122a007df", + "request_id": "98025f84-ad34-4fca-a4fa-84b5aefad08d" + }, + { + "utc": 1795102473.8665411, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005139667016919702, + "trace_id": "30bac7df1fdb790766c3e97766b26298", + "request_id": "f029b548-4f2d-4819-946b-9da61e6be57f" + }, + { + "utc": 1795102474.363971, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005206792033277452, + "trace_id": "f2044698b082aad5f0f680c042f28302", + "request_id": "a7240682-eab5-4014-8727-43a9bb848709" + }, + { + "utc": 1795102474.867473, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0072395000024698675, + "trace_id": "623ccf141eef4712f13cad0ca6116940", + "request_id": "f50e8eec-a7be-49b9-900d-481cbc8b24bc" + }, + { + "utc": 1795102475.363369, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007532707997597754, + "trace_id": "429afde67204a9965d00517d9326151d", + "request_id": "93f0818f-fe70-44c0-b8bc-3027bc2382be" + }, + { + "utc": 1795102475.3634062, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.012688415998127311, + "trace_id": "2efe516e7832c45320e2640275b1c733", + "request_id": "2671afef-dc04-4689-8f2f-ccbc381ab473" + }, + { + "utc": 1795102475.8640552, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005822665989398956, + "trace_id": "e50135c43a76a8cabddb632e65f7ad78", + "request_id": "b7ab0bf2-7886-4cf6-b46f-e8734d5ce8b4" + }, + { + "utc": 1795102476.362452, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.004929291026201099, + "trace_id": "7298be83374af2586ae51c42cc738c3e", + "request_id": "06c84dcf-7a71-42cf-864b-b011ebc976f2" + }, + { + "utc": 1795102476.864326, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005459291045553982, + "trace_id": "ccec2cadaecf63a841cdc5ee3403d43e", + "request_id": "f58180c6-fc79-4dd7-b9c4-d44894c2ea05" + }, + { + "utc": 1795102477.363342, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007589666987769306, + "trace_id": "e63a0e1d28ff929cc6ad68c509da6377", + "request_id": "fd668405-0a22-4ad8-9ec0-3918af39852b" + }, + { + "utc": 1795102477.363373, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011619500000961125, + "trace_id": "438b11d5c76187874ff692374e8f3613", + "request_id": "e896c8a2-0a79-4398-bc95-da8508b7b3dd" + }, + { + "utc": 1795102477.867442, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00565262499731034, + "trace_id": "2ed8b8ee7f92ab9e08f1d7af1fb4df67", + "request_id": "940064c5-8cf9-45c1-a91c-83b698d5f224" + }, + { + "utc": 1795102478.363394, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005639666051138192, + "trace_id": "f5d04280a1d6ce98a29aafc5ce294128", + "request_id": "8c345ee6-6d7e-4ee2-8273-cc6e2e65f0b9" + }, + { + "utc": 1795102478.8642628, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008190374996047467, + "trace_id": "4ce80a7785cb3b2e7cc2a0b6f47dcaa8", + "request_id": "a8c8ec7d-13db-44b0-87d0-859d3c12b7f8" + }, + { + "utc": 1795102479.362942, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005973834020551294, + "trace_id": "33e56b2ccebe3d89e812ff0edbef549a", + "request_id": "94970253-1384-4179-aa0f-e4d1032870dc" + }, + { + "utc": 1795102479.362962, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.010389709030278027, + "trace_id": "49eff90c36ae5e43980df39f3e4fb718", + "request_id": "81788e0f-5c1e-4d39-8e6a-95243c8b4c48" + }, + { + "utc": 1795102479.8670552, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008876792038790882, + "trace_id": "0b192127a924f8fd399cefffed12642a", + "request_id": "1085de18-ffc3-4842-ae74-30095e56c06e" + }, + { + "utc": 1795102480.367428, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0058768330491147935, + "trace_id": "d6dc06988c676679bb72a379bfb88f53", + "request_id": "ceffa697-9d17-442e-8600-9ea0916e247a" + }, + { + "utc": 1795102480.862586, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005689249956049025, + "trace_id": "cf0faf75ec9a32f9ee91481a8d77b592", + "request_id": "d72855ce-7270-4605-bd23-c4753b0d6c28" + }, + { + "utc": 1795102481.3638048, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0065763749880716205, + "trace_id": "c4d7effe3dac33535c57fe247703e270", + "request_id": "c778c037-917a-4c26-a290-29879bff93f1" + }, + { + "utc": 1795102481.3638358, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01183424994815141, + "trace_id": "e8e318f7dd0d88518ca8cae2989fcd30", + "request_id": "8fddc521-feb9-4cf5-9346-8ad266d7e193" + }, + { + "utc": 1795102481.8627589, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005534083000384271, + "trace_id": "cc5befeb0614ba28942b3f96782634e3", + "request_id": "a51265b3-2e1e-4e57-a741-0a134ec335fd" + }, + { + "utc": 1795102482.364366, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00564779102569446, + "trace_id": "48f4400e91872334813d52f71772be18", + "request_id": "1df80b4f-0d45-4a0d-9dac-ac280b11de9e" + }, + { + "utc": 1795102482.8637178, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006344125024043024, + "trace_id": "203e44f13a3c193dc7a33dfeefb95e65", + "request_id": "92fd223f-e716-4938-9e68-62e1be19c49c" + }, + { + "utc": 1795102483.364352, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007637790986336768, + "trace_id": "62bc3c8a2df6a37fc2d8678c9f68899e", + "request_id": "aca1fae3-ebc4-4080-a8a5-fd8b8f73335a" + }, + { + "utc": 1795102483.364382, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011511542019434273, + "trace_id": "76ebbb417eb4d1380b73d1514c6abb85", + "request_id": "1f4e8a3c-29f1-4072-b892-4b195f456b7c" + }, + { + "utc": 1795102483.8674371, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.004967292014043778, + "trace_id": "cf38d8268ed5d020fde91a0c22544567", + "request_id": "c0874905-a9af-4a1c-96cd-290a6b829fa2" + }, + { + "utc": 1795102484.3630779, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006031207973137498, + "trace_id": "165f0725342e94e5cb02b6d766b0cf65", + "request_id": "c5a38c3a-70b0-422e-b9df-cb6740de8cc2" + }, + { + "utc": 1795102484.8673658, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007552957977168262, + "trace_id": "bd40cb1cd50ac1944ec6dbd1c55fc0d4", + "request_id": "68dd6afa-fd83-4273-9058-ba2a05436c2e" + }, + { + "utc": 1795102485.362744, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0070729589788243175, + "trace_id": "84f3e42691d423677ca3aff553f0aa56", + "request_id": "c29bdc75-4e93-4a15-a5af-58358128038e" + }, + { + "utc": 1795102485.3627741, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011691999970935285, + "trace_id": "629b950ceb3aaf53161c7293178ddf2c", + "request_id": "fa879f5d-ccc2-4fe1-834e-981f4197c866" + }, + { + "utc": 1795102485.862588, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005988291988614947, + "trace_id": "a0a7390d7ccf69800f6a2920125c814a", + "request_id": "77033f83-85a8-4129-ae61-3d688ceb0ad6" + }, + { + "utc": 1795102486.363163, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007752041972707957, + "trace_id": "ca595c1b7ac0dda086bfb99bed254c0f", + "request_id": "98c38407-03f2-4b31-8f88-bfe6a8afabbc" + }, + { + "utc": 1795102486.862596, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0065709580085240304, + "trace_id": "1ca576cf54ac231c8e298a078ed81c0a", + "request_id": "5d8964e2-31e8-46ca-b89b-71dbeade12ab" + }, + { + "utc": 1795102487.363245, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018868417013436556, + "trace_id": "a7bffe3b749a48a885d18da3c48bbd71", + "request_id": "2958597d-d1f8-4013-aeec-21fe250bcc06" + }, + { + "utc": 1795102487.36329, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030525749956723303, + "trace_id": "83a2d5839b960f7120e4118da41376a4", + "request_id": "bebf0ae5-d255-49e3-98d2-d82ecedbcf7a" + }, + { + "utc": 1795102487.8675141, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0067389170289970934, + "trace_id": "1e4ab0390fc6f253bb10f6ac481cd9e0", + "request_id": "9e1bdfdd-6bd5-43b7-8b4d-e02fb264077f" + }, + { + "utc": 1795102488.367519, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008541207993403077, + "trace_id": "20f04ebe529a17e338017f838d703432", + "request_id": "c8917895-7f50-4162-94f7-1784bab93c28" + }, + { + "utc": 1795102488.863731, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01526441698661074, + "trace_id": "735504ad3f75e2e65315039093939294", + "request_id": "a21507ca-ee0e-41c2-8576-db586035bb08" + }, + { + "utc": 1795102489.364203, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0238905840087682, + "trace_id": "b6b8f1917953aeac468f5c3a4a582f3a", + "request_id": "d3643b21-d858-42f0-934f-73b307b79800" + }, + { + "utc": 1795102489.364235, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03229479200672358, + "trace_id": "1aea33167b21328ace22f26297471cc7", + "request_id": "ae558699-89a2-4092-9b90-1379230d2afb" + }, + { + "utc": 1795102489.867423, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006087457993999124, + "trace_id": "305b1c80b68b7f047bb5604172f3b2c9", + "request_id": "9344cee1-3c21-4fb5-bea8-148943900987" + }, + { + "utc": 1795102490.3638651, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008001207956112921, + "trace_id": "8d731bb0c6a9d916f877c7a71a3c2593", + "request_id": "bc4ffd8d-6f98-4a54-9418-8915cf4e8c8a" + }, + { + "utc": 1795102490.8674932, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.020759583043400198, + "trace_id": "52d8275b2dcb5b5a9cead30c1e6c04d8", + "request_id": "9038908b-e380-4417-a245-74e09509826b" + }, + { + "utc": 1795102491.362578, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01747845800127834, + "trace_id": "ddd10377a2765741a9c94a8ce040a28f", + "request_id": "036991fa-ec6b-45e9-a0b1-20d5148b504f" + }, + { + "utc": 1795102491.362619, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029111167008522898, + "trace_id": "36a8a7761277ad8206b511638c53b897", + "request_id": "40791aee-9d74-4c14-8ae7-7cde06ca28bf" + }, + { + "utc": 1795102491.864341, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0052632910083048046, + "trace_id": "5a515b9e88df106e23228a9b4cb4efa5", + "request_id": "091ba9ef-e2e9-406d-8a33-c91c960e3f55" + }, + { + "utc": 1795102492.367475, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.010575333959423006, + "trace_id": "21bcc740ddbd45988de4446ff4b68232", + "request_id": "a7d026b0-1c48-4e0f-963e-e399607f10a8" + }, + { + "utc": 1795102492.867488, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005781667015980929, + "trace_id": "7eff0c9c8650eec7f900a31074e7ee56", + "request_id": "0fcc9ad1-0e0c-4ce6-8b65-fc9085630dd5" + }, + { + "utc": 1795102493.364743, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017209292040206492, + "trace_id": "c0f9747f115ef6580baec04ac7057f75", + "request_id": "4b31faed-c8d5-4102-9aef-a4cc6fa26b2f" + }, + { + "utc": 1795102493.364775, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02979770803358406, + "trace_id": "cb2b7f665f17e1091874902898466add", + "request_id": "4b8c0b8a-16de-41dc-b5ac-ccdfc9fef344" + }, + { + "utc": 1795102493.863713, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007984958007000387, + "trace_id": "4aacfb0bf145002716d10932c4a7ed72", + "request_id": "ba9ef84c-098d-4681-920b-7b37188b4e1e" + }, + { + "utc": 1795102494.3633559, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008627625007648021, + "trace_id": "f3ca811d1f876fe132f0144e8786fb91", + "request_id": "aa1930b0-e29c-4188-9c8e-5d2e50caa937" + }, + { + "utc": 1795102494.865437, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00988091598264873, + "trace_id": "90a6f0b8fcf106fe4db1f77a4fbe671f", + "request_id": "2986785b-e06f-4383-b9da-11313881982a" + }, + { + "utc": 1795102495.3674822, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015115791000425816, + "trace_id": "50468d0e2d55cd764fe90a1f9df27055", + "request_id": "6baef8cd-6e9c-4717-ac70-a0e3d74b60ef" + }, + { + "utc": 1795102495.3675258, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.026321540994103998, + "trace_id": "ff7a187de87c2c106f41bfbba276a9f1", + "request_id": "e8d9b70d-daa7-43d0-95c8-98691ec4f5b9" + }, + { + "utc": 1795102495.867483, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007638375042006373, + "trace_id": "15f8044c364f6fe37615a8fbaba28f84", + "request_id": "dddac4bf-1631-4fd4-944c-f91015690729" + }, + { + "utc": 1795102496.367002, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.011338833021000028, + "trace_id": "c9c190c8cfbdc20bd5f3bbb9296715da", + "request_id": "19bdbf43-55db-4805-9708-97aa057f4cc9" + }, + { + "utc": 1795102496.867485, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009285874955821782, + "trace_id": "5467d2ebfcf2a139a39a46f1b142a49e", + "request_id": "84df745b-cbb4-458b-9a31-58abac1fab05" + }, + { + "utc": 1795102497.365315, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016855374968145043, + "trace_id": "83750083d7e5204fd1e40a17723fcb79", + "request_id": "4cd0dfad-8f3c-458c-bde8-70bdffa6d2b3" + }, + { + "utc": 1795102497.365346, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.027967250032816082, + "trace_id": "bb1cae5fac849ac6b64d35eaa8bd3aa3", + "request_id": "0724d3e2-1078-4987-a446-475c1541c78b" + }, + { + "utc": 1795102497.862922, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005911707994528115, + "trace_id": "b5475e75a207efa1115eb80289fd369a", + "request_id": "f4886197-b5e0-4633-8310-cae9e68e2c0e" + }, + { + "utc": 1795102498.3632262, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.025110125017818063, + "trace_id": "26a51a12bbe93941eb4e3f2410d955dc", + "request_id": "b5b85ded-86bf-48c4-be74-6016b6acf6ff" + }, + { + "utc": 1795102498.863631, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018005333025939763, + "trace_id": "8ed04d299c7b33a60f19a45280c5ee27", + "request_id": "2ee59b51-4b2b-491f-aaf7-7c816d115198" + }, + { + "utc": 1795102499.3637938, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007426874944940209, + "trace_id": "eeae6ba79211187ed2b1e42ed4d31cdf", + "request_id": "757bea7e-95f0-4f98-b489-6f8e4bced903" + }, + { + "utc": 1795102499.3638499, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011703875032253563, + "trace_id": "ebde2c3d72f20e649f866de13d165994", + "request_id": "1d020294-fee5-4ade-92ca-08638753dac5" + }, + { + "utc": 1795102499.863376, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0055303749977611005, + "trace_id": "89df95caddbba933c6c38833cb231b1f", + "request_id": "28f0285a-e7f0-4d44-8ed2-3f33ee1b54fa" + }, + { + "utc": 1795102500.3644412, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017446374986320734, + "trace_id": "309f7e9368ba4cabea069d30adb69695", + "request_id": "5fad1091-b6d0-47d8-ab81-0400353b4363" + }, + { + "utc": 1795102500.867382, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013257584010716528, + "trace_id": "d60874cf6abb0fd9d408c8f74c3e6bca", + "request_id": "f105628e-f920-4e91-ba38-0aaf3016832f" + }, + { + "utc": 1795102501.367331, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013965708028990775, + "trace_id": "2fceb5f89c986c4850328b68bfd327c5", + "request_id": "f344a7ed-6485-4e9f-9752-ebdfe637d69d" + }, + { + "utc": 1795102501.367359, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.021071541996207088, + "trace_id": "2bfc54ddab5a91d42b610222527ff21e", + "request_id": "5288c8d5-b1a5-44bd-aa89-f3d76cc33909" + }, + { + "utc": 1795102501.865609, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005691083963029087, + "trace_id": "488a3a5f4f76a818e0c9e306e7b01b0c", + "request_id": "07951a38-c6b1-42d3-918f-9f533c8f1905" + }, + { + "utc": 1795102502.3674932, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016237500007264316, + "trace_id": "c39ec8be2b493813b57e4c0994367adb", + "request_id": "4542358d-577a-4245-944f-19a0973e7ff4" + }, + { + "utc": 1795102502.863938, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00888991600368172, + "trace_id": "1dc0bd029d3c798a6c497119793c0b54", + "request_id": "66bd3071-39c8-41a9-a359-09f9df0fcfd5" + }, + { + "utc": 1795102503.3633568, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.019462917000055313, + "trace_id": "44cdc3335c9c7c593b2cbf07d08091c4", + "request_id": "ce8dd103-b7d3-4278-8a9d-8bc836b01cce" + }, + { + "utc": 1795102503.363432, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030749209050554782, + "trace_id": "1e4a937b9fb08cc2632c5399cbb8b708", + "request_id": "17143d91-5d24-44a1-9df8-d43838014d2e" + }, + { + "utc": 1795102503.867316, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0059933330048806965, + "trace_id": "b22db5c2dfb28cdab439360073911c11", + "request_id": "8a728382-4810-4102-bb37-59aa7ea269a3" + }, + { + "utc": 1795102504.3648791, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01639624999370426, + "trace_id": "df981a78ee963f6e9748a84d2c823ae1", + "request_id": "bf7d0c08-266f-42fb-b50d-65ea07bb7d65" + }, + { + "utc": 1795102504.863342, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009178207954391837, + "trace_id": "96dc2b7c8074c2dd91ad0f7e1950cbca", + "request_id": "31f32a3b-1a29-4ba0-9492-92cc2d2c23ab" + }, + { + "utc": 1795102505.365126, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01845441700424999, + "trace_id": "8be76481fbc7a111af18fb56699975f8", + "request_id": "ccd99968-226d-4a8c-ab35-1412e9af1768" + }, + { + "utc": 1795102505.365171, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.028776958002708852, + "trace_id": "9909fb5f2f184aa66cc8b767b3218dbf", + "request_id": "de5b8b15-2ad2-470f-be49-30c7108a3f23" + }, + { + "utc": 1795102505.863242, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005278165976051241, + "trace_id": "be8dca404b083e4028bd5a0a434d85cd", + "request_id": "85d642f6-e4a2-4790-8ae5-db73d719e32a" + }, + { + "utc": 1795102506.367421, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015934124996419996, + "trace_id": "74d2442d645fe88effeb7369026e251b", + "request_id": "cf7646c6-9fad-4aa3-8b71-ece22e0581de" + }, + { + "utc": 1795102506.863107, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008144207997247577, + "trace_id": "6043dab003d09fdb8c37a5ba08ee6631", + "request_id": "80b26ee7-6866-4b74-af46-e514c750f981" + }, + { + "utc": 1795102507.364094, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016587916994467378, + "trace_id": "584e4fd1a0e13de50a5021a3c2f42e10", + "request_id": "872775aa-f6fe-40e6-a1d6-c080264f393f" + }, + { + "utc": 1795102507.364139, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02771283296169713, + "trace_id": "389815d463930f21dcb558760090ccbf", + "request_id": "3fea1971-f913-4db7-b947-cbbeae60402f" + }, + { + "utc": 1795102507.8673549, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007908249972388148, + "trace_id": "3399c769738bbea274676bac0ba16367", + "request_id": "5bc196b0-54eb-4d8f-9798-58fe4d6cdc24" + }, + { + "utc": 1795102508.363306, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016596416011452675, + "trace_id": "bd8bedfc058403510548ed2334865306", + "request_id": "b1d2a194-2c37-45c8-ab5e-6eee16b23cac" + }, + { + "utc": 1795102508.8638568, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.019069833972025663, + "trace_id": "8b9f8b034ad89f799aee6987dd1c957f", + "request_id": "5d52384b-158c-458f-a053-2fd34cd71b6e" + }, + { + "utc": 1795102509.3674471, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018569500010926276, + "trace_id": "ae4ee13043b4b3c67e2c26f2edb0066e", + "request_id": "a0e4c81b-6f73-4e0f-aa9e-f2e27c24b55b" + }, + { + "utc": 1795102509.3674922, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03058133297599852, + "trace_id": "ee3ea4816e2870beb7d02ffe65e463cc", + "request_id": "f0be4513-a9cb-4def-9441-df73028d8ffa" + }, + { + "utc": 1795102509.867402, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006595459009986371, + "trace_id": "79b5f7a46cef14a2477038dd8aa69a06", + "request_id": "e691aa61-61b4-41b8-9546-c2400e0ed9d7" + }, + { + "utc": 1795102510.36742, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015841124986764044, + "trace_id": "c6faf13624db69d89aaebebb9183c2fa", + "request_id": "9aaf2352-c593-4907-87fa-042c2c3368ae" + }, + { + "utc": 1795102510.86739, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016844000027049333, + "trace_id": "60dac910a57ac4158b9b7504f3064e15", + "request_id": "ba5463d6-f10e-47d8-ac3b-98184d39c54c" + }, + { + "utc": 1795102511.3650172, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01808912499109283, + "trace_id": "366e52c266271b41a7589c3d1f3ffbe8", + "request_id": "557976a6-b717-428a-a216-c1e66734c45e" + }, + { + "utc": 1795102511.365043, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02963033301057294, + "trace_id": "9bd808daf582d33be8da6df50d66dcb5", + "request_id": "110f6bdd-506a-4db1-9e61-3408cfb37260" + }, + { + "utc": 1795102511.866532, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005522874998860061, + "trace_id": "0fea3e5fc7f824914304aed6de5d2612", + "request_id": "e68098cd-5fd1-4fc1-9919-13f1749aac27" + }, + { + "utc": 1795102512.36361, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007852792041376233, + "trace_id": "9410ed3b27e8850232f8438e7eac6199", + "request_id": "b46ede73-ca2a-47ec-ac65-9a9ddac297b8" + }, + { + "utc": 1795102512.863273, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.014764583960641176, + "trace_id": "16a674ca6edbfd1dc50e4249f453bc19", + "request_id": "a1ce39e7-cc9e-4350-9605-de845871b602" + }, + { + "utc": 1795102513.367362, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.026718791981693357, + "trace_id": "65c84f7a88677dfaa0a6e509f41609da", + "request_id": "c74bb069-2df2-42ca-9995-2befee0345b7" + }, + { + "utc": 1795102513.3674061, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.036125458020251244, + "trace_id": "907bc1295f9950d53ff68940d31b27d5", + "request_id": "59c4a4e4-77ad-47bd-b73d-27c81771faa9" + }, + { + "utc": 1795102513.863227, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005720416957046837, + "trace_id": "eb6e1660be7d4d9348bb4d3a45c4ffa1", + "request_id": "c6d3039b-b0f3-48c9-b385-52233815eb6d" + }, + { + "utc": 1795102514.364269, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005719250009860843, + "trace_id": "7c8f579d5061936357645e9e91ab5f0b", + "request_id": "7e8df83e-b218-4679-89bb-e55ee4f4b629" + }, + { + "utc": 1795102514.863939, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007884000020567328, + "trace_id": "53b30b29cdb07f553c74eacab8477ad3", + "request_id": "2a1a905c-ea26-4ba9-9172-c58fb740945b" + }, + { + "utc": 1795102515.36301, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017966374987736344, + "trace_id": "3d4d033fe75789ade3c810b1466f091b", + "request_id": "8a3ece8c-310e-4b5e-a6b0-ae48209ba3c6" + }, + { + "utc": 1795102515.363053, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03020004101563245, + "trace_id": "e7e25b7bea5fe4616905cf109ebc0f42", + "request_id": "fa6319e7-5218-4906-bd29-400d45db95ab" + }, + { + "utc": 1795102515.864161, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0061054579564370215, + "trace_id": "69ef478dfb9b1cd3b135600851de04c8", + "request_id": "3de6144b-c497-45b5-9094-955c966f4d91" + }, + { + "utc": 1795102516.364722, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018602666968945414, + "trace_id": "375f0fbc140e5961a5a1097a07814464", + "request_id": "e8d8ec3e-f144-4097-8c95-074f29fe28b1" + }, + { + "utc": 1795102516.8632782, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007857832999434322, + "trace_id": "5e47b2ad7daeb58819e9788377683480", + "request_id": "b7e3a844-2637-4c55-b68a-ced16073a981" + }, + { + "utc": 1795102517.367146, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013653500005602837, + "trace_id": "c201b4575d631b655cd190658060608b", + "request_id": "0def77de-a2b2-48dd-afdb-572dbe1bd631" + }, + { + "utc": 1795102517.367203, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.023083875013981014, + "trace_id": "46c6e2847298fcd7c28b4b8abb8809b8", + "request_id": "0486b807-2115-4009-a479-463ca6f2ac0d" + }, + { + "utc": 1795102517.86324, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007990750018507242, + "trace_id": "6ceedac60785f7f3dd3273d3263fd831", + "request_id": "e8174e0c-c920-4e63-8dfb-b4eccf746f91" + }, + { + "utc": 1795102518.367388, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016281584044918418, + "trace_id": "320f7e68724e3751214c3c12198c3626", + "request_id": "7a8b5f13-447f-4c4f-86ed-76b4ddb492a1" + }, + { + "utc": 1795102518.863341, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007526334025897086, + "trace_id": "7b43cae3c270ccb390adb235f8f8a289", + "request_id": "e2b9cca0-5bc7-4148-b997-d9b287f4c202" + }, + { + "utc": 1795102519.367429, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007834124960936606, + "trace_id": "072c9d6bad75372f05f5b3679b3230d4", + "request_id": "a784d066-c776-46f2-8f2a-1cab4c2926d3" + }, + { + "utc": 1795102519.367468, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.014225374965462834, + "trace_id": "07bed6e9b032fe880573a91474aafff3", + "request_id": "bc230b9b-f379-48f5-9beb-edc4b714dbe0" + }, + { + "utc": 1795102519.863283, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007632458000443876, + "trace_id": "66904d4b12ba6d700405cee6df2d5e5a", + "request_id": "10b4ceca-f9aa-4bfb-beeb-f143d7f48031" + }, + { + "utc": 1795102520.363317, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017928082961589098, + "trace_id": "2c3addee6c370d8d69821ebd99e2dfb3", + "request_id": "50d2a667-cbef-4090-b751-9841aae01780" + }, + { + "utc": 1795102520.8673081, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005574250011704862, + "trace_id": "eec379461b933e1a4d437c1824ba0b57", + "request_id": "5d9529e4-ffd0-4a00-8840-797620979b59" + }, + { + "utc": 1795102521.363034, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016927250020671636, + "trace_id": "a902e37d8b5abf91fa0ede6ec50f81bd", + "request_id": "13041239-d334-4a19-ad60-519513d0bcff" + }, + { + "utc": 1795102521.363061, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.026232250034809113, + "trace_id": "d48a6834abc51dd28dd17c9db8a9159a", + "request_id": "63ac5004-16b5-4852-a499-e06f4f352c8d" + }, + { + "utc": 1795102521.867399, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0062610839959234, + "trace_id": "ce8fca8c483506e49c3c0e8a3cba9f6e", + "request_id": "0bd252ec-ccb1-49c1-b6c0-56881821df5a" + }, + { + "utc": 1795102522.367388, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.021573624981101602, + "trace_id": "b4fc0d8b24cb5eb46c9f2ddd51d60e69", + "request_id": "5acfa07d-c18d-4557-925e-254802b5b304" + }, + { + "utc": 1795102522.863355, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006978167046327144, + "trace_id": "4daa8dd40db6f5fa97ba4746574c752f", + "request_id": "cd86a328-6d0c-4ff1-aa8a-ccfa009b670a" + }, + { + "utc": 1795102523.3647912, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017361167003400624, + "trace_id": "cae5945cc0f4a4c8b1485e0adaf8113a", + "request_id": "762252f6-ea5c-4046-8b89-259bd56f507e" + }, + { + "utc": 1795102523.364831, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029835916007868946, + "trace_id": "9de39d9c66fe65cc1f71d679ee559811", + "request_id": "b3523a6d-169b-4ce8-9cad-81a169f4ec5f" + }, + { + "utc": 1795102523.867394, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00643679202767089, + "trace_id": "66f33bb2a7e39d7cfcbb0afab0ebcaf8", + "request_id": "54f1b661-ca52-47e5-bb0f-7a295de9ad8c" + }, + { + "utc": 1795102524.3670158, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013598458026535809, + "trace_id": "568fd5d6cc5c453987347437ccee5932", + "request_id": "9f3f343a-a9aa-41e9-a867-9dbd76e235a4" + }, + { + "utc": 1795102524.8635242, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.010301250033080578, + "trace_id": "16db2c9bee9f43feb9b793f9181ec01f", + "request_id": "2e75ad45-1aa1-466f-88c9-762cdbd909a8" + }, + { + "utc": 1795102525.363486, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007639417017344385, + "trace_id": "cc470e35779a5975e1b44b5f6afa0e42", + "request_id": "84425395-a897-42c5-aa74-8b3555cf1992" + }, + { + "utc": 1795102525.363533, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.013148042024113238, + "trace_id": "02427def1ffa9e98eaef392f3c95db4b", + "request_id": "1e9025f0-1d7d-4ece-a94d-da1339e47728" + }, + { + "utc": 1795102525.8672671, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0055042500025592744, + "trace_id": "29df6d32dc98a44f5587b973c72c6b91", + "request_id": "683548d0-6245-45cb-8664-1065cce4b955" + }, + { + "utc": 1795102526.362666, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017041999963112175, + "trace_id": "a008d957e723340f7a957d32b3f8f5b2", + "request_id": "f1b4a86d-3c8d-4a51-af7b-3f11ed59fa59" + }, + { + "utc": 1795102526.8672872, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008198875002563, + "trace_id": "106ee7686de85f66e11368aa73611b12", + "request_id": "06de42ef-c5fb-46ae-ba54-35d83792c7f1" + }, + { + "utc": 1795102527.3623838, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006745500024408102, + "trace_id": "e7753994fc28aef580fcdeb4bf7f536a", + "request_id": "baf4450f-b71a-4162-9d00-e914390f62b8" + }, + { + "utc": 1795102527.362429, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011576250020880252, + "trace_id": "72fc8312e734cc46f4a9cb5220b88bd4", + "request_id": "065aeef9-e124-40b1-847a-d49f4e664173" + }, + { + "utc": 1795102527.863152, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0051940419944003224, + "trace_id": "a00c9fa7dc2fe2aca7f0b26c7377364a", + "request_id": "ddfc6d40-ed78-40e0-af61-1ee8d839ab8f" + }, + { + "utc": 1795102528.366112, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016452374984510243, + "trace_id": "8a4b89fbe888062b8ca226f46e415c85", + "request_id": "8a778069-b0ce-4e9b-88ad-ff33a60eb9b5" + }, + { + "utc": 1795102528.866407, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.020821915997657925, + "trace_id": "078adb5c7bf8ea1ac8e08cb907e6c6bf", + "request_id": "ffce939a-f166-4723-92e0-4f66a23d0e73" + }, + { + "utc": 1795102529.367381, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018863749981392175, + "trace_id": "bd024bb5d3c48e989b5c67efaa09f65a", + "request_id": "ec4912bb-9054-438d-a9ae-9348c2001e02" + }, + { + "utc": 1795102529.367428, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03192283300450072, + "trace_id": "514908b98fe485720f31b7d97ab419a7", + "request_id": "849eb513-9247-4179-8394-b3d54b743b12" + }, + { + "utc": 1795102529.867282, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005619457981083542, + "trace_id": "ab0ab7c2c1f2fc29b4a2dd6fb6abf90a", + "request_id": "e3857067-e4ca-4a66-a31e-b805cd05f411" + }, + { + "utc": 1795102530.3641942, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.014652084035333246, + "trace_id": "755cf8b3d67d74b62152e92ee00eddf4", + "request_id": "aa7a7393-872f-42fe-afa8-d0c1c64fdd0c" + }, + { + "utc": 1795102530.867413, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016687916999217123, + "trace_id": "7e46c727df4cfc9e70bd6c1e11db843c", + "request_id": "d05f7f3d-a5f9-437e-a2de-301760e5a2d4" + }, + { + "utc": 1795102531.365052, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0183074590167962, + "trace_id": "0081e38e7945c626cf29d8566f2a6b0b", + "request_id": "677ab4e5-6b29-43fd-9557-e876ebd6d349" + }, + { + "utc": 1795102531.365091, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030797791958320886, + "trace_id": "fe54f6787db15fc5084a1a57d15448a4", + "request_id": "a6f2ece0-5a98-43fc-a35e-d7087b98902d" + }, + { + "utc": 1795102531.867369, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006972167000640184, + "trace_id": "4771651aa2be7c4c8b8bb11606cb1119", + "request_id": "56ef93bc-487b-49b9-affd-930156530114" + }, + { + "utc": 1795102532.36417, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016096875013317913, + "trace_id": "2967e7b4d22b568e274dfe73d4eec472", + "request_id": "324e5a1b-5f39-4a94-acd6-f591a45192b6" + }, + { + "utc": 1795102532.867348, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013620665995404124, + "trace_id": "a27a1f60ce60e4ee2d55da6f37e52f45", + "request_id": "0437d45a-adf4-4429-b336-d582daa3dbf5" + }, + { + "utc": 1795102533.36736, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018275166978128254, + "trace_id": "038dfc03e920b15444a0546dd811ddec", + "request_id": "a27113d0-834c-4392-9c00-8dfe8b350685" + }, + { + "utc": 1795102533.367405, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029854374995920807, + "trace_id": "c7ef965a43516c5f62a0e819acabb944", + "request_id": "f29359f2-657f-44d5-aca8-45678be919f4" + }, + { + "utc": 1795102533.8680582, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00567154202144593, + "trace_id": "7fc063c6ce42652e8e3f76a0589837a2", + "request_id": "5bb169f6-a50c-4f2e-9a89-56be660656b7" + }, + { + "utc": 1795102534.363497, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01883625000482425, + "trace_id": "dc1826797a6b4bcb36a310a5dffe8ede", + "request_id": "c052a986-ecef-4f52-89a5-89f68af191bb" + }, + { + "utc": 1795102534.868064, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009922042023390532, + "trace_id": "f6b1b2b55e5f0f60432e5eee6294456a", + "request_id": "17706bcb-fc8a-493c-b837-b1f710a6f509" + }, + { + "utc": 1795102535.36732, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.019955625000875443, + "trace_id": "bd9e70baba4c8892d6fb1dbd3ab11da2", + "request_id": "f4cb83b0-9388-43f2-a233-f52e8289e034" + }, + { + "utc": 1795102535.367363, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.031611874990630895, + "trace_id": "d257f6676a9eb4fca625509d6b8cabd2", + "request_id": "52e9d02d-b655-4a7b-8fe2-a7391335e505" + }, + { + "utc": 1795102535.868897, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007162166992202401, + "trace_id": "6bc8e33a023ac71a7bc6dac2ded0ff4e", + "request_id": "9a2147bf-3d83-4280-bc92-862aa67e4a6c" + }, + { + "utc": 1795102536.368035, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01944133301731199, + "trace_id": "bdd1bfa42c4296cce6b585a84a8af077", + "request_id": "43579e65-6f76-4b3e-8460-e3ae1b53c0f0" + }, + { + "utc": 1795102536.869998, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009179165994282812, + "trace_id": "1fd2b9f73a72c4ae51966364f1abc66c", + "request_id": "aa6a9955-9a0f-4c0c-ac0f-f439dc3ce3f0" + }, + { + "utc": 1795102537.372223, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.021086207998450845, + "trace_id": "092d17aa1c58cc9342c179b71f6de8f6", + "request_id": "046e1ccc-a442-4f20-81ee-e3b2d3bb8ca2" + }, + { + "utc": 1795102537.372264, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03110229200683534, + "trace_id": "bf3c670b76c60664c2432d3ca3ea537c", + "request_id": "8d925153-1159-4dc5-9918-4231010f463e" + }, + { + "utc": 1795102537.869948, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006308416021056473, + "trace_id": "a536f7ed02e6781a9c3ca827c8c8c950", + "request_id": "837c9846-777c-4a8e-9546-b61c2615689f" + }, + { + "utc": 1795102538.364415, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017329958034679294, + "trace_id": "1750ad5c62afa5b7eca299ffd9335655", + "request_id": "c7debec3-519f-45d1-8d95-1445de5d322d" + }, + { + "utc": 1795102538.870024, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017548082978464663, + "trace_id": "d2e9296ff669009151b6402229d03be6", + "request_id": "096d5d48-40ab-44d5-8acc-ed6c541a66fc" + }, + { + "utc": 1795102539.370052, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01763312501134351, + "trace_id": "29d86e0e164e238e5d3ff72b60e0cc67", + "request_id": "201659e2-e5de-4618-afd8-de9bad33926a" + }, + { + "utc": 1795102539.370097, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030750540958251804, + "trace_id": "94406ba19666d26ae6463cb5a16d79a4", + "request_id": "bfae7605-c065-46c3-9fc8-dc5c91a68843" + }, + { + "utc": 1795102539.8723352, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006503415992483497, + "trace_id": "02ddd3ae49af34826301d069be426bec", + "request_id": "1c9b76d6-7f0f-4979-81a7-8dade189d993" + }, + { + "utc": 1795102540.3700302, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00943145900964737, + "trace_id": "a966da9f22c404d9db085945cde871eb", + "request_id": "b52a5720-f779-4319-ac1c-7c5e1980bc4a" + }, + { + "utc": 1795102540.870016, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016296334040816873, + "trace_id": "3ebc30953f3c3b9815863f81637b179d", + "request_id": "d88fff3d-f7b1-42e2-92f0-4c58ab5bdd18" + }, + { + "utc": 1795102541.368022, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017248000018298626, + "trace_id": "c55edb7f25a3133cea728e34004d3996", + "request_id": "b53b3f6d-0c94-4e86-9d14-4282ed5172ff" + }, + { + "utc": 1795102541.368049, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02662545902421698, + "trace_id": "57f483a506fcd9caf955b9b6e3ba9c33", + "request_id": "b0c311c0-4c18-42da-a7a8-65d9408669e0" + }, + { + "utc": 1795102541.872229, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005301167024299502, + "trace_id": "50856759ac44d97adea768b11bab64d1", + "request_id": "76631e3f-245b-4838-9817-25de963489b7" + }, + { + "utc": 1795102542.372239, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005781707994174212, + "trace_id": "b6f839b1ae3597a9ce0af13553547762", + "request_id": "16f25130-3801-40ae-9b63-f22e1a5385bb" + }, + { + "utc": 1795102542.863282, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01897204201668501, + "trace_id": "7ec7719dedc939f9cc03c56e7b5e673c", + "request_id": "ee44b63e-34fc-4d50-9250-63571d8e1a02" + }, + { + "utc": 1795102543.366863, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013889416004531085, + "trace_id": "0db1e229afdf96831dca8b35d40f0c15", + "request_id": "35d6aea1-75a3-4df0-9cdf-e4ef500ce13b" + }, + { + "utc": 1795102543.3668768, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.022566833009477705, + "trace_id": "d2119b0b69bc27c9512275f85d7ec2d0", + "request_id": "cf3d3dcd-c567-42b3-9f9e-6745479b6d5f" + }, + { + "utc": 1795102543.869787, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005083124968223274, + "trace_id": "41380598f95732ac0075190ace977088", + "request_id": "c3d9157b-73c7-4193-baf7-450649f49ddf" + }, + { + "utc": 1795102544.3723001, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016708250041119754, + "trace_id": "b49f47b9351fc8e36fd8281bd2b64a31", + "request_id": "1ceafd99-d80f-461d-8256-db9f29da5f62" + }, + { + "utc": 1795102544.872295, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009809457988012582, + "trace_id": "8415b042a99a8b04ad27b922ef489cc4", + "request_id": "b6ccc5ee-0ba5-4a21-8e06-3039d511e47f" + }, + { + "utc": 1795102545.364863, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008590332989115268, + "trace_id": "832d1b948925d55e1ddb44ae8641fed5", + "request_id": "e6be07c0-b920-4964-a5db-89be89b5fcbf" + }, + { + "utc": 1795102545.3650408, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01313395902980119, + "trace_id": "8786cde53b6d0fc7e0d1d1e71bdef397", + "request_id": "4d783cab-834f-4f2c-8f2a-ac0b85ed1c7d" + }, + { + "utc": 1795102545.867934, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005082042014691979, + "trace_id": "2fda2fa652d6ef6f1f2f5ddbd8632799", + "request_id": "2ae44961-4bee-43d1-8eb1-d3acd02e0284" + }, + { + "utc": 1795102546.370284, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01688900002045557, + "trace_id": "6968ff37a61b4b03ea5cccaa9a168fcd", + "request_id": "2cfcbad3-9dd2-4e69-957d-903e208d32bc" + }, + { + "utc": 1795102546.8723881, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007037999981548637, + "trace_id": "1caca842a794b06327aab3bec9c49ca6", + "request_id": "5cf2ad4a-db3e-4ae0-b4a7-bbcb75062e25" + }, + { + "utc": 1795102547.363092, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01658950001001358, + "trace_id": "7fbfad2f1deac5d4db6962fedf3b621b", + "request_id": "fe568efa-f944-4d1a-8358-f694f498a000" + }, + { + "utc": 1795102547.36312, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02544875000603497, + "trace_id": "7416c110768b9cb16090bfe4c91f5c3e", + "request_id": "95f62af2-ed6c-4166-82b3-863169cf1830" + }, + { + "utc": 1795102547.8686888, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006563582981470972, + "trace_id": "1ef7f89084b63778978a47647f50aae3", + "request_id": "9d72397d-1404-45b9-a874-c7e780431828" + }, + { + "utc": 1795102548.3699892, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01651404198491946, + "trace_id": "fb8c1090a7fb2b68cc16feb91c9d9c83", + "request_id": "d26afc00-ddfb-49ea-a161-a5bf9d8e0632" + }, + { + "utc": 1795102548.870374, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00544495799113065, + "trace_id": "045e78dc61743e327c8bbbc13179e070", + "request_id": "e287baac-debc-47c3-9c30-a5dd96d5b20b" + }, + { + "utc": 1795102549.365428, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.010001624992582947, + "trace_id": "5c05cd04484876583654e89a87265377", + "request_id": "5d105cb9-3264-4307-8030-3c79ed2dd1de" + }, + { + "utc": 1795102549.365473, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01957254100125283, + "trace_id": "c6bc5533926773805ab46926948c4197", + "request_id": "9dd6205e-2d17-49fe-b9a0-40c94ab9bbe2" + }, + { + "utc": 1795102549.868682, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008493583009112626, + "trace_id": "6bf0d580dc49ef6986eb0ca45bbaf45d", + "request_id": "cb69a179-48c5-4421-9da1-e88aad845924" + }, + { + "utc": 1795102550.367166, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017261749948374927, + "trace_id": "56d23892a353b53462c52f9cef23dc06", + "request_id": "80b697d5-12e5-425c-a3e5-2ff11126cc86" + }, + { + "utc": 1795102550.868001, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016706000023987144, + "trace_id": "ca376fa568f19babe277188f28eaddef", + "request_id": "0111bfe3-28df-4bfc-9328-18cb7c4c2b39" + }, + { + "utc": 1795102551.369795, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016741750005166978, + "trace_id": "07eabd52154fb9d8a90b057964b69da4", + "request_id": "0dcf89bf-5788-4e98-9ac5-2b8ee2827672" + }, + { + "utc": 1795102551.3698409, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.025790125015191734, + "trace_id": "479af2e84397e5c258d6f295b6422a59", + "request_id": "eb261ecd-340e-4836-8515-032d0d9cbf7a" + }, + { + "utc": 1795102551.8680599, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007134749961551279, + "trace_id": "0ea1ff1d9ec9629b2e4f24847de91bc2", + "request_id": "94b4bc3d-cd60-45b6-8786-0e6dcb6db4fa" + }, + { + "utc": 1795102552.372322, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016893082996830344, + "trace_id": "adde9cb090906633a45c2265835848c3", + "request_id": "9e42ddb4-0a21-41d7-bbfa-d84527444bad" + }, + { + "utc": 1795102552.864921, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0198257919983007, + "trace_id": "001d665e09b5a97776e22658838c06ed", + "request_id": "7e60b741-0222-4762-b79f-cc900c6f4444" + }, + { + "utc": 1795102553.368002, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008378958038520068, + "trace_id": "b3ec4b01214326e372c33fce1db0d196", + "request_id": "7aac9857-2e2b-4402-9c22-c252b27482fb" + }, + { + "utc": 1795102553.368051, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.014631458034273237, + "trace_id": "bdb9f9165b67751346d23206e0906c37", + "request_id": "9531c56b-c470-4ecb-8e31-5a8ec8c7fce6" + }, + { + "utc": 1795102553.8699598, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.006110707996413112, + "trace_id": "ad193714aaebd8a954580ce93b5f6d88", + "request_id": "74f134ec-5e4a-4715-a3a3-f328c2ebdab4" + }, + { + "utc": 1795102554.372222, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015187791956122965, + "trace_id": "71d6d4c50897d7f8f6583a0a87ff7b98", + "request_id": "1e12e20e-75b2-4469-b81a-b3f401bc6e49" + }, + { + "utc": 1795102554.87009, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.010168000007979572, + "trace_id": "ac5cffb2248edeb8502303d7d823ad9a", + "request_id": "c01fde43-ee23-4dd9-aebf-c4dd4786c8a9" + }, + { + "utc": 1795102555.3721962, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008272790990304202, + "trace_id": "24bbc59db72f38a1e0cceebaf1b6301f", + "request_id": "a4516224-3213-47b5-abd1-6a6fc9872e77" + }, + { + "utc": 1795102555.372227, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.012843541975598782, + "trace_id": "67ada7445946878a6978f1f69c9025d0", + "request_id": "f779080b-2d57-4309-9cd5-e310da67c6d0" + }, + { + "utc": 1795102555.872298, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0052575000445358455, + "trace_id": "cef90b964f831230a2efda1c218ce7e9", + "request_id": "aea344e5-cd18-4df1-8a51-22fa1ff275a3" + }, + { + "utc": 1795102556.36995, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008428666973486543, + "trace_id": "1337dc2ead80b74cd622d1561f45708e", + "request_id": "77587f99-2c2b-47d0-9fba-3459d7e32389" + }, + { + "utc": 1795102556.867949, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008916874998249114, + "trace_id": "ec63d491041301b0596c27847a4b37fe", + "request_id": "1e546e2a-916a-4eb8-b5bf-6d5f8d9677e0" + }, + { + "utc": 1795102557.372339, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.026137790991924703, + "trace_id": "8ac76fc4314945d65f00e2050f47ad60", + "request_id": "195c4d45-fd3f-4595-88a2-46f4dd11c7a9" + }, + { + "utc": 1795102557.3723829, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03527183400001377, + "trace_id": "e288e273594bb1be62e2305102527676", + "request_id": "37868972-020c-436c-a83e-bb76b82ccdc5" + }, + { + "utc": 1795102557.867889, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005309125001076609, + "trace_id": "4cf862ea31071ea384668158f13ca8eb", + "request_id": "96ac5625-9961-4dfd-83a3-352aacc329be" + }, + { + "utc": 1795102558.3629088, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00858374999370426, + "trace_id": "32509408d7c9e4994e6a0cfb03b9870d", + "request_id": "373213c3-3d1c-46a5-a66d-5d0111abce89" + }, + { + "utc": 1795102558.869312, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015766499971505255, + "trace_id": "3df0975644ea54f9eeec09025d3cfd35", + "request_id": "1792c52d-9cf0-4200-9ffe-39dfce1dea5b" + }, + { + "utc": 1795102559.369944, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.02015624998603016, + "trace_id": "a3ebe31537d98f893ec30051d7277497", + "request_id": "27ec00cf-b797-4af8-a9d5-1929874bc68f" + }, + { + "utc": 1795102559.3699882, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030821708030998707, + "trace_id": "d5ce66733da38d286fd340289160e582", + "request_id": "fc77ca8f-870f-4113-9cac-b2fb28a9c481" + }, + { + "utc": 1795102559.867883, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00518066604854539, + "trace_id": "1947ebfcd9292b62aa54cf11f872643f", + "request_id": "21c28f36-f888-472e-8355-10a37dbb7eda" + }, + { + "utc": 1795102560.372299, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.020493916992563754, + "trace_id": "49dbf3a80d02e1eb852aca102115bbcc", + "request_id": "04f907c2-27ae-4b29-8a58-bae3c87691b6" + }, + { + "utc": 1795102560.864104, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013987332989927381, + "trace_id": "3b9819525c6089ae50ff2531182797d4", + "request_id": "b6bc4755-709b-4420-8792-d41b34a4cd16" + }, + { + "utc": 1795102561.3679738, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01696004200493917, + "trace_id": "33dc81d378283a1f1ea226a414d04d24", + "request_id": "040da8f4-0660-4f66-bb87-ebec3c7b3538" + }, + { + "utc": 1795102561.3680189, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02888187498319894, + "trace_id": "b498ef56abe279dc8d44c0ebca127e5e", + "request_id": "9dbcb734-1888-4670-80ea-e2e930fd332b" + }, + { + "utc": 1795102561.869805, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005985499999951571, + "trace_id": "9b3b83f877e08edd73ffb794af8571aa", + "request_id": "4ebc3436-eb07-4a3c-b85e-4f42d1d4693d" + }, + { + "utc": 1795102562.3639019, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.014682083041407168, + "trace_id": "a2f8012393e168847d0a0013723bc740", + "request_id": "872cbd71-40bf-435b-a5bd-2eed669bbcd3" + }, + { + "utc": 1795102562.8679512, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0171147080254741, + "trace_id": "3914cf1286a2aa0240fa9366e3441c33", + "request_id": "e5134ee2-6936-4cef-84e2-4ba37ca11920" + }, + { + "utc": 1795102563.368733, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.024210083996877074, + "trace_id": "bcc82762671566b481601a3c7786e832", + "request_id": "80b3cc44-3f65-43a9-927b-6aa9909ba772" + }, + { + "utc": 1795102563.3688471, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03669741697376594, + "trace_id": "3a2a5ac172b7ab5b09410e51a02dc6fa", + "request_id": "84ec5771-bc8f-452d-a449-c60b9e4ddbd2" + }, + { + "utc": 1795102563.864032, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005745458998717368, + "trace_id": "dfaf5c71d982b294c9a6d26897a26213", + "request_id": "8a5a0549-6e40-46d0-8ee8-85db057942bc" + }, + { + "utc": 1795102564.367907, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015080749988555908, + "trace_id": "553cc776e2c5b1448fe3ede18121b90d", + "request_id": "d57117a8-935d-49e3-9693-2688856b3501" + }, + { + "utc": 1795102564.869906, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008629999996628612, + "trace_id": "92a2da1ad01b4a14ccbe193979d6b0e0", + "request_id": "4e933cf9-35c8-46b5-bc87-9df1b6e1981f" + }, + { + "utc": 1795102565.3692968, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01870862499345094, + "trace_id": "4b8a399c23b4e842dfb661c1ecb244a6", + "request_id": "d06d446b-985d-4dfd-86f6-f3ca9dfb6107" + }, + { + "utc": 1795102565.36934, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03012454102281481, + "trace_id": "02ee936951837bf4fce4c63c6dd54004", + "request_id": "55677312-4f56-4649-8102-b201841c486c" + }, + { + "utc": 1795102565.87216, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005433582991827279, + "trace_id": "7eb4d8e3ba18b86efa5f1a137d0533c5", + "request_id": "c994f634-4393-4e86-9717-3ee1bfa12e7c" + }, + { + "utc": 1795102566.369926, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.011315999960061163, + "trace_id": "134781872cffdbafb242f11d1e42466a", + "request_id": "82ca08c7-6160-4c3a-a18e-9af3000a993f" + }, + { + "utc": 1795102566.864071, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.008603667025454342, + "trace_id": "597dee0a8d3d80f0001e8d1fa9d456b9", + "request_id": "d9260241-d917-4a81-83f2-2cbee5e30047" + }, + { + "utc": 1795102567.372229, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.020681166963186115, + "trace_id": "9a30662465fe5bf299091c77ed8c8654", + "request_id": "079d2724-b873-45bb-8277-0ec26c5c36f1" + }, + { + "utc": 1795102567.372267, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03335250000236556, + "trace_id": "99805ced04a9b8afb09d88c9e2fa49f8", + "request_id": "aa7fb085-ba22-4daa-b8a7-894fed4e5668" + }, + { + "utc": 1795102567.869812, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.004943749983794987, + "trace_id": "9fcff05e9087f573307f0437a5d66794", + "request_id": "34fbd30d-4988-45f3-9d92-77cc1d91f0bf" + }, + { + "utc": 1795102568.369853, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005479499988723546, + "trace_id": "e22e04ca63662d4ce97ce54a0e577690", + "request_id": "65b862d3-9e5a-40d1-b6c4-95d023f98631" + }, + { + "utc": 1795102568.867933, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018230375018902123, + "trace_id": "3afc6e0bcf1b8f9bf020c2788cba691e", + "request_id": "707b1e03-f818-4dd9-a413-0dd90d8637ab" + }, + { + "utc": 1795102569.369917, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.020866500039119273, + "trace_id": "a6d3d06e6e7d79606bb31a5caf525521", + "request_id": "0c076433-aeae-451f-bc23-b6e410ed07af" + }, + { + "utc": 1795102569.3699641, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03346883394988254, + "trace_id": "67e4c913e333f54374cf7ea4536d0c92", + "request_id": "76f8db53-3a09-4124-a8b4-bf3fe3350a7c" + }, + { + "utc": 1795102569.867848, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005378125002607703, + "trace_id": "52a4953b4d727038b808909f232959d3", + "request_id": "d4b6ca50-5511-4df2-b4d0-5d902709d7fa" + }, + { + "utc": 1795102570.363331, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.012724957952741534, + "trace_id": "288554aef3c4a95d913a160822536ffb", + "request_id": "50f2bbcc-f0e2-4fa2-9adc-5daf8ab20e68" + }, + { + "utc": 1795102570.869886, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.011996583023574203, + "trace_id": "329fd3fab91cb2cf51916cedcdcfe504", + "request_id": "f9342e1e-6db7-489c-8e3c-85504ba0bfbd" + }, + { + "utc": 1795102571.37141, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.019850957964081317, + "trace_id": "ca361f0edcef198513cf5fd45d22e437", + "request_id": "f9a3b110-a5d6-4cfc-ba1b-01c6ebdb9aa9" + }, + { + "utc": 1795102571.371454, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.03256829199381173, + "trace_id": "f5e8a7fa95397640b5a72be46712ac33", + "request_id": "3488cada-fd05-4953-ae79-98863af30013" + }, + { + "utc": 1795102571.872221, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0064381249831058085, + "trace_id": "8bba4e2977e7b72f7a6d1513d13ac8d5", + "request_id": "15451a59-d7c2-41ac-84fc-02d2cfbb17fb" + }, + { + "utc": 1795102572.3661509, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016636749962344766, + "trace_id": "50960ce94d4a3c55cb2b326cf819a032", + "request_id": "0b628ef8-15dd-4c7d-b6cf-50f0f1810300" + }, + { + "utc": 1795102572.862874, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.011382917000446469, + "trace_id": "7b6c2d52766fe701712f6ab0bdd6bb49", + "request_id": "0216c7ed-432f-4f37-9acf-a64ebb7e436c" + }, + { + "utc": 1795102573.366828, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01886879204539582, + "trace_id": "892d05f2a7a543313c0c34e7124eeb66", + "request_id": "d3632769-b6a1-45a3-9bca-2e91b131c355" + }, + { + "utc": 1795102573.366875, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.030406000034417957, + "trace_id": "f4e06a4793d1594bc70dd5e4116cddbd", + "request_id": "6b2cac5b-edc0-4d89-8a74-af000bcbc6f5" + }, + { + "utc": 1795102573.872086, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.00666312500834465, + "trace_id": "e7d907a3620c221cb751a72d81c290bb", + "request_id": "3caf70cf-c74f-4bfe-af20-92578403fb20" + }, + { + "utc": 1795102574.3699021, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.016235416987910867, + "trace_id": "53da57d32507fef8bd61ee511276b3ec", + "request_id": "6a70880f-09bc-4436-ab6b-20f7d90e183a" + }, + { + "utc": 1795102574.864052, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007280041987542063, + "trace_id": "452b76a188707a8f766086a5b02aa075", + "request_id": "145b6917-2133-43b7-b967-431f34b5f479" + }, + { + "utc": 1795102575.3679428, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01904804102377966, + "trace_id": "4e7e82ea7e89f2dfaf02a8a7f958f560", + "request_id": "85e1c253-427e-4cb0-ac09-80dcfc56a574" + }, + { + "utc": 1795102575.367983, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.029937124985735863, + "trace_id": "a3fdf01497cf8c3c60c8281ec1dce81c", + "request_id": "2d10362d-bf4f-4ba7-96f5-81b491ce3e87" + }, + { + "utc": 1795102575.863838, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005290083994623274, + "trace_id": "3bbf0ed4b9fbc0da6927ee720e14f031", + "request_id": "ad8f0d6e-4832-415c-ae11-5c6e6b905d3d" + }, + { + "utc": 1795102576.366337, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013628250046167523, + "trace_id": "d95746e97715c5f0e8686c068acd6c23", + "request_id": "bb3a2465-9461-4127-b1a1-93fab6e8978f" + }, + { + "utc": 1795102576.868309, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0060779579798690975, + "trace_id": "8e6a508631d614e74f04df40e44778c2", + "request_id": "66c509f3-b62f-405e-9bf7-1b5ed51749ed" + }, + { + "utc": 1795102577.370553, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018274165980983526, + "trace_id": "8e2f9c6f8fe515edfaabc437a27cd482", + "request_id": "6116e490-a5a0-4cff-adcb-48a9a0cb1f70" + }, + { + "utc": 1795102577.370592, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02902745798928663, + "trace_id": "8323c59ab9cedfa4854a7094ad3f4318", + "request_id": "55f02409-b053-4551-bb27-5e655e62b29a" + }, + { + "utc": 1795102577.8684978, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0064112909603863955, + "trace_id": "92eec9fa9a072ec8c5263468b24f540e", + "request_id": "92cb8b18-bdac-40ed-8d64-06e03818595f" + }, + { + "utc": 1795102578.3690722, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0169244579738006, + "trace_id": "71c3df9edb84bf1198624ca3563e8c3f", + "request_id": "ea001970-d1b7-4794-8117-b63d78492b50" + }, + { + "utc": 1795102578.872214, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017118124989792705, + "trace_id": "7ed7fab33aff875f7acbe2cfb12c5315", + "request_id": "397f54b0-e2fb-4b1f-826a-36adace92d4f" + }, + { + "utc": 1795102579.367873, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017833500052802265, + "trace_id": "3cb3b14fcb9021a10e9bb73129dc247a", + "request_id": "419f594d-605e-4cdd-93f2-646d281a5338" + }, + { + "utc": 1795102579.367908, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02813550003338605, + "trace_id": "d2e392c83d59895f84ed2ea2c30ed369", + "request_id": "8bff01cd-19b5-4722-aec4-67b6bb3b5cb2" + }, + { + "utc": 1795102579.8678389, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005494999990332872, + "trace_id": "a31de59caae91c0ef1241e00e947c440", + "request_id": "58915ab4-0077-4e44-9f49-07bb050b7b50" + }, + { + "utc": 1795102580.362255, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.012519082985818386, + "trace_id": "7f1bb8c2bcc0591a791d85175a8b3311", + "request_id": "0192ef39-1e58-4501-9c5d-3030d7d78e35" + }, + { + "utc": 1795102580.872159, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.018585417012218386, + "trace_id": "bdf2a6f74115f7111181009004a2fd36", + "request_id": "0f15d7d1-4644-4085-901a-0808259d194d" + }, + { + "utc": 1795102581.3689559, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007307082996703684, + "trace_id": "e4b5083c5fe7c53c1e846da9ac637546", + "request_id": "11de6fa9-f64c-4fea-b4cb-48ae78a3e4e5" + }, + { + "utc": 1795102581.368974, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.012241666961926967, + "trace_id": "a39382c26cb9502238f476e781b13297", + "request_id": "35263ada-7365-4a3a-afad-77184146b7d3" + }, + { + "utc": 1795102581.8679109, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01025087502785027, + "trace_id": "9723c556a0b126604ecb8c01238ae673", + "request_id": "b919cef2-603c-494b-81fc-2b020aecd875" + }, + { + "utc": 1795102582.367441, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017959208984393626, + "trace_id": "3e7182bb7f138e8f6c1cc706ff6212e3", + "request_id": "85653bd8-43c8-41a0-802a-3ef421f66ae9" + }, + { + "utc": 1795102582.869845, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01791908300947398, + "trace_id": "92b66d7b0da26f02bc45d3244a8092ec", + "request_id": "399abbd2-b3c7-4464-b375-d11b7703a5d2" + }, + { + "utc": 1795102583.3678038, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007109042024239898, + "trace_id": "0969b1a6ebc14d8425f3de720e01cd9f", + "request_id": "3a9e8917-8854-49a2-b32a-f1325b80c1f4" + }, + { + "utc": 1795102583.367834, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.011675834015477449, + "trace_id": "6711b42fe7f6ab2f6cd720d2119e1816", + "request_id": "f170ffcc-0fa3-41a3-ac64-145030b05152" + }, + { + "utc": 1795102583.8679051, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.011094500019680709, + "trace_id": "13f9c30300787dadf2c3dc54cb46a7e3", + "request_id": "98790d3c-ed58-44f9-b3e0-4bcb1b766106" + }, + { + "utc": 1795102584.36401, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017894291959237307, + "trace_id": "c6e1d9676a0e090c337cd0c9a4d898ec", + "request_id": "66f403f0-bb94-45c9-b640-ad675c77f8cd" + }, + { + "utc": 1795102584.872184, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007141458976548165, + "trace_id": "d9a7fc113b39176358815ef6454a4e0e", + "request_id": "b6723a6e-ae12-407d-8400-247f35198236" + }, + { + "utc": 1795102585.364612, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.013001249986700714, + "trace_id": "aa68c5f2b5ce2cd405dc15a423f8bdb9", + "request_id": "0951b40f-316d-4c52-8869-115752b8a26e" + }, + { + "utc": 1795102585.364652, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.0180504999589175, + "trace_id": "3f217f324c4efdec868ef28a230f8970", + "request_id": "0f9ebea2-bd74-4d6b-b47b-64a049378732" + }, + { + "utc": 1795102585.867824, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005622166965622455, + "trace_id": "b5d372f37e283a8baa5774a7e244a99e", + "request_id": "cc7e3237-b781-4807-a98a-df4e8123a0e7" + }, + { + "utc": 1795102586.367871, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.014838708972092718, + "trace_id": "68e42faac12cd3c241923b495ccd848e", + "request_id": "1bd2158d-da34-4122-a123-02cb9f3ebeb3" + }, + { + "utc": 1795102586.869848, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.007331917004194111, + "trace_id": "26c09cef7d0cef639feb95999f93e17d", + "request_id": "eb69ca47-59bd-401b-b7fd-49d7edb2b1ad" + }, + { + "utc": 1795102587.3639958, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.010997540957760066, + "trace_id": "d49a076bb9a38c16f039cfc061c297d0", + "request_id": "c6777865-fd5c-4aa1-b071-4753342c90d9" + }, + { + "utc": 1795102587.3640409, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.01633750001201406, + "trace_id": "d3932f55a0eb3d0e70b1145e88b1bacc", + "request_id": "461ac341-62a5-4db3-a7ae-3e7b13fad772" + }, + { + "utc": 1795102587.864017, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009787582966964692, + "trace_id": "c0b16112c7f37cc612f6c8fa5139fc35", + "request_id": "4931e7b2-70dd-4541-bd1a-6210b0484473" + }, + { + "utc": 1795102588.3678908, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.015340458019636571, + "trace_id": "f835796f72ed00a382b6c080286790d7", + "request_id": "8a5a3f12-e8e6-4748-a7b8-151f465b10ac" + }, + { + "utc": 1795102588.8631291, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.017034207994583994, + "trace_id": "232b13e7d2bb2a05a5000d00d96e59bf", + "request_id": "33a66359-c0e5-4d68-a48b-064347b93cda" + }, + { + "utc": 1795102589.369774, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.014223791018594056, + "trace_id": "488db401c97541f91f683fb2df9c66ca", + "request_id": "d46a4c26-fbf2-4a06-9ad7-612f11bca1e0" + }, + { + "utc": 1795102589.369798, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.02203987498069182, + "trace_id": "d96ec9b0257f033035b20fea4ac9d94a", + "request_id": "bbe3de32-023a-48f2-ac37-0525a09c26bd" + }, + { + "utc": 1795102589.869812, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.005714749975595623, + "trace_id": "37c700ada2f2164194dae6d249ccea6e", + "request_id": "a339f2db-d8f8-4d3b-aa8e-17ac19387b61" + }, + { + "utc": 1795102590.369231, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.012973584001883864, + "trace_id": "706b2414b01f699fa534a0cb74d437c2", + "request_id": "1efa5d72-044b-4917-8bff-b00b2e4af3a9" + }, + { + "utc": 1795102590.869806, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.009593375027179718, + "trace_id": "279f75f5b932eec551dd76f1b97499b2", + "request_id": "29be774f-906c-4c3b-b0b0-52d94ff48f97" + }, + { + "utc": 1795102591.367828, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.022843167011160403, + "trace_id": "f2edb2a973e41e72968af4d389751a26", + "request_id": "b03c3680-c5a9-4552-b5d5-f4c64456195d" + }, + { + "utc": 1795102591.367857, + "phase": "after_worker_recovery", + "kind": "upload", + "status": 202, + "duration_seconds": 0.0316198340151459, + "trace_id": "09c50b5fe629a0608b51ea219c378a29", + "request_id": "46b8c924-3d01-44ea-8cfa-bb7a3a69ff00" + }, + { + "utc": 1795102591.867769, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.0047586250002495944, + "trace_id": "a65a239fd4b040ed05f213386b8559c0", + "request_id": "a270feeb-bda7-4fe5-88aa-31def053d1fa" + }, + { + "utc": 1795102592.3719769, + "phase": "after_worker_recovery", + "kind": "message", + "status": 201, + "duration_seconds": 0.01571783301187679, + "trace_id": "4767b482736a8ef25c9a99dad9938a0d", + "request_id": "53cc7e6a-1212-408b-8a75-7950ad69a46c" + } + ], + "skipped_arrivals": 0, + "phase_summary": { + "baseline": { + "requests": 301, + "5xx": 0, + "successful_rps": 2.5081891707489476, + "p50_ms": 16.607, + "p95_ms": 31.266, + "p99_ms": 35.193 + }, + "short_db_spike": { + "requests": 10, + "5xx": 4, + "successful_rps": 1.4962418299487805, + "p50_ms": 1670.714, + "p95_ms": 2017.188, + "p99_ms": 2017.188 + }, + "after_short_spike": { + "requests": 150, + "5xx": 0, + "successful_rps": 2.499795602583431, + "p50_ms": 16.475, + "p95_ms": 30.739, + "p99_ms": 32.763 + }, + "sustained_db_lock": { + "requests": 225, + "5xx": 176, + "successful_rps": 0.5444424728476727, + "p50_ms": 2011.805, + "p95_ms": 2022.545, + "p99_ms": 2025.758 + }, + "after_db_recovery": { + "requests": 250, + "5xx": 0, + "successful_rps": 2.4999512562719017, + "p50_ms": 16.283, + "p95_ms": 30.965, + "p99_ms": 33.443 + }, + "worker_stopped": { + "requests": 300, + "5xx": 0, + "successful_rps": 2.4999259036886072, + "p50_ms": 16.221, + "p95_ms": 31.277, + "p99_ms": 35.499 + }, + "after_worker_recovery": { + "requests": 300, + "5xx": 0, + "successful_rps": 2.4999904185900736, + "p50_ms": 13.134, + "p95_ms": 30.749, + "p99_ms": 35.272 + } + }, + "alert_timings": { + "APIAvailabilitySlowBurn": { + "detection_seconds": 60.28, + "resolution_after_restore_seconds": 65.27 + }, + "APIHighErrorRate": { + "detection_seconds": 25.11, + "resolution_after_restore_seconds": 35.15 + }, + "APIAvailabilityFastBurn": { + "detection_seconds": 40.18, + "resolution_after_restore_seconds": 35.15 + }, + "AttachmentQueueLag": { + "detection_seconds": 45.21, + "resolution_after_restore_seconds": 30.12 + }, + "AttachmentReadyFastBurn": { + "detection_seconds": 95.5, + "resolution_after_restore_seconds": 45.18 + }, + "AttachmentReadySlowBurn": { + "detection_seconds": 131.6, + "resolution_after_restore_seconds": 75.32 + } + }, + "worker_diagnosis": { + "trace_id": "2b1592ca18aac08bbd14b3f1825f0816", + "request_id": "bad9f74b-1237-475e-b7a9-13e04b9457fb", + "accept_to_consume_seconds": 94.29, + "consumer_after_restore": true + } +} diff --git a/evidence/lab6-slo-research.txt b/evidence/lab6-slo-research.txt new file mode 100644 index 0000000..7fab4df --- /dev/null +++ b/evidence/lab6-slo-research.txt @@ -0,0 +1,246 @@ +PHASE: baseline 120 seconds +PHASE: short_db_spike 4 seconds +PHASE: after_short_spike 60 seconds +PHASE: sustained_db_lock 90 seconds +PHASE: after_db_recovery 100 seconds +PHASE: worker_stopped 120 seconds +PHASE: after_worker_recovery 120 seconds +{ + "utc": "2026-11-19T15:26:14.987289+00:00", + "hypothesis": "Multiwindow burn alerts ignore a 4s spike, detect sustained DB/worker degradation and resolve after recovery.", + "profile": { + "message_arrival_per_second": 2, + "image_arrival_per_second": 0.5, + "max_inflight": 16, + "sampling": 1.0, + "retries": 0, + "slo_window_seconds": 3600, + "api_slo": 0.99, + "ready_slo": 0.95, + "ready_deadline_seconds": 30 + }, + "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", + "db_diagnosis": { + "trace_id": "f124d93f84dd25c5af978e78e2028646", + "request_id": "3c5e5b14-5825-41b8-a227-93cba420a532", + "slow_error_spans": [ + { + "name": "postgresql.SELECT", + "status": { + "code": "STATUS_CODE_ERROR" + }, + "duration_ms": 2000.115335, + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "_WaitTimeout" + } + } + ] + }, + { + "name": "postgresql.transaction", + "status": { + "code": "STATUS_CODE_ERROR" + }, + "duration_ms": 2001.169168, + "attributes": [ + { + "key": "error.type", + "value": { + "stringValue": "_WaitTimeout" + } + } + ] + }, + { + "name": "POST /api/v1/chats/{chat_id}/messages", + "status": { + "code": "STATUS_CODE_ERROR" + }, + "duration_ms": 2003.784959, + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/messages" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "503" + } + } + ] + } + ] + }, + "short_spike_observed_error_ratio_peak": 0.06373490138007629, + "passed": true, + "false_positives": { + "baseline": 0, + "burn_alerts_after_short_spike": 0, + "single_window_alerts_after_short_spike": [ + "APIHighErrorRate" + ] + }, + "start": 1795101974.987257, + "end": 1795102594.377363, + "duration_seconds": 619.3901059627533, + "phases": [ + { + "name": "baseline", + "start": 1795101975.364327, + "planned_seconds": 120, + "active_at": 1795101975.364415, + "end": 1795102095.3712242 + }, + { + "name": "short_db_spike", + "start": 1795102095.371294, + "planned_seconds": 4, + "active_at": 1795102095.527463, + "end": 1795102099.381341 + }, + { + "name": "after_short_spike", + "start": 1795102099.381383, + "planned_seconds": 60, + "active_at": 1795102099.381435, + "end": 1795102159.386289 + }, + { + "name": "sustained_db_lock", + "start": 1795102159.386304, + "planned_seconds": 90, + "active_at": 1795102159.552147, + "end": 1795102249.3866298 + }, + { + "name": "after_db_recovery", + "start": 1795102252.4302201, + "planned_seconds": 100, + "active_at": 1795102252.430262, + "end": 1795102352.43217 + }, + { + "name": "worker_stopped", + "start": 1795102352.4322171, + "planned_seconds": 120, + "active_at": 1795102352.766314, + "end": 1795102472.4357738 + }, + { + "name": "after_worker_recovery", + "start": 1795102472.435835, + "planned_seconds": 120, + "active_at": 1795102474.322918, + "end": 1795102592.4362948 + } + ], + "skipped_arrivals": 0, + "phase_summary": { + "baseline": { + "requests": 301, + "5xx": 0, + "successful_rps": 2.5081891707489476, + "p50_ms": 16.607, + "p95_ms": 31.266, + "p99_ms": 35.193 + }, + "short_db_spike": { + "requests": 10, + "5xx": 4, + "successful_rps": 1.4962418299487805, + "p50_ms": 1670.714, + "p95_ms": 2017.188, + "p99_ms": 2017.188 + }, + "after_short_spike": { + "requests": 150, + "5xx": 0, + "successful_rps": 2.499795602583431, + "p50_ms": 16.475, + "p95_ms": 30.739, + "p99_ms": 32.763 + }, + "sustained_db_lock": { + "requests": 225, + "5xx": 176, + "successful_rps": 0.5444424728476727, + "p50_ms": 2011.805, + "p95_ms": 2022.545, + "p99_ms": 2025.758 + }, + "after_db_recovery": { + "requests": 250, + "5xx": 0, + "successful_rps": 2.4999512562719017, + "p50_ms": 16.283, + "p95_ms": 30.965, + "p99_ms": 33.443 + }, + "worker_stopped": { + "requests": 300, + "5xx": 0, + "successful_rps": 2.4999259036886072, + "p50_ms": 16.221, + "p95_ms": 31.277, + "p99_ms": 35.499 + }, + "after_worker_recovery": { + "requests": 300, + "5xx": 0, + "successful_rps": 2.4999904185900736, + "p50_ms": 13.134, + "p95_ms": 30.749, + "p99_ms": 35.272 + } + }, + "alert_timings": { + "APIAvailabilitySlowBurn": { + "detection_seconds": 60.28, + "resolution_after_restore_seconds": 65.27 + }, + "APIHighErrorRate": { + "detection_seconds": 25.11, + "resolution_after_restore_seconds": 35.15 + }, + "APIAvailabilityFastBurn": { + "detection_seconds": 40.18, + "resolution_after_restore_seconds": 35.15 + }, + "AttachmentQueueLag": { + "detection_seconds": 45.21, + "resolution_after_restore_seconds": 30.12 + }, + "AttachmentReadyFastBurn": { + "detection_seconds": 95.5, + "resolution_after_restore_seconds": 45.18 + }, + "AttachmentReadySlowBurn": { + "detection_seconds": 131.6, + "resolution_after_restore_seconds": 75.32 + } + } +} diff --git a/evidence/lab6-slo-series.json b/evidence/lab6-slo-series.json new file mode 100644 index 0000000..0b106ee --- /dev/null +++ b/evidence/lab6-slo-series.json @@ -0,0 +1,8354 @@ +{ + "http_rps": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {}, + "values": [ + [ + 1795101974.987, + "0" + ], + [ + 1795101979.987, + "12.220978159541882" + ], + [ + 1795101984.987, + "17.92391304347826" + ], + [ + 1795101989.987, + "16.245408389992445" + ], + [ + 1795101994.987, + "15.525625000000002" + ], + [ + 1795101999.987, + "15.154984604310792" + ], + [ + 1795102004.987, + "14.998986246278273" + ], + [ + 1795102009.987, + "14.929182855292726" + ], + [ + 1795102014.987, + "14.945879059070572" + ], + [ + 1795102019.987, + "14.989219259259258" + ], + [ + 1795102024.987, + "15.081275000000002" + ], + [ + 1795102029.987, + "15.177853636363634" + ], + [ + 1795102034.987, + "8.673473469833073" + ], + [ + 1795102039.987, + "2.49113555777798" + ], + [ + 1795102044.987, + "2.5091821520782576" + ], + [ + 1795102049.987, + "2.4905921064591783" + ], + [ + 1795102054.987, + "2.5094102885821834" + ], + [ + 1795102059.987, + "2.490773230551061" + ], + [ + 1795102064.987, + "2.5092734017019422" + ], + [ + 1795102069.987, + "2.4909090909090907" + ], + [ + 1795102074.987, + "2.509090909090909" + ], + [ + 1795102079.987, + "2.4909090909090907" + ], + [ + 1795102084.987, + "2.509090909090909" + ], + [ + 1795102089.987, + "2.4909090909090907" + ], + [ + 1795102094.987, + "2.509090909090909" + ], + [ + 1795102099.987, + "2.4909090909090907" + ], + [ + 1795102104.987, + "2.509410288582183" + ], + [ + 1795102109.987, + "2.490399403712255" + ], + [ + 1795102114.987, + "2.5087984679344992" + ], + [ + 1795102119.987, + "2.4903541303083188" + ], + [ + 1795102124.987, + "2.5090314017128206" + ], + [ + 1795102129.987, + "2.4907163636363636" + ], + [ + 1795102134.987, + "2.5088981818181817" + ], + [ + 1795102139.987, + "2.4907163636363636" + ], + [ + 1795102144.987, + "2.5088981818181817" + ], + [ + 1795102149.987, + "2.4903541303083188" + ], + [ + 1795102154.987, + "2.5088981818181817" + ], + [ + 1795102159.987, + "2.4903541303083188" + ], + [ + 1795102164.987, + "2.5094102885821834" + ], + [ + 1795102169.987, + "2.454947173173792" + ], + [ + 1795102174.987, + "2.436275044543835" + ], + [ + 1795102179.987, + "2.4177861804432" + ], + [ + 1795102184.987, + "2.436230751049943" + ], + [ + 1795102189.987, + "2.4179180453041487" + ], + [ + 1795102194.987, + "2.436363636363636" + ], + [ + 1795102199.987, + "2.41783013379872" + ], + [ + 1795102204.987, + "2.436275044543835" + ], + [ + 1795102209.987, + "2.41783013379872" + ], + [ + 1795102214.987, + "2.4363636363636365" + ], + [ + 1795102219.987, + "2.4180059632026762" + ], + [ + 1795102224.987, + "2.4725924040506886" + ], + [ + 1795102229.987, + "2.4909090909090907" + ], + [ + 1795102234.987, + "2.509364657962687" + ], + [ + 1795102239.987, + "2.4908638024763188" + ], + [ + 1795102244.987, + "2.509364657962687" + ], + [ + 1795102249.987, + "2.4909090909090907" + ], + [ + 1795102254.987, + "2.509455920861216" + ], + [ + 1795102259.987, + "2.563822823478071" + ], + [ + 1795102264.987, + "2.582193773639802" + ], + [ + 1795102269.987, + "2.5638694426766073" + ], + [ + 1795102274.987, + "2.582005964070114" + ], + [ + 1795102279.987, + "2.5637762059748717" + ], + [ + 1795102284.987, + "2.582287688670667" + ], + [ + 1795102289.987, + "2.563776205974871" + ], + [ + 1795102294.987, + "2.6001891046621575" + ], + [ + 1795102299.987, + "2.5818181818181816" + ], + [ + 1795102304.987, + "2.6" + ], + [ + 1795102309.987, + "2.5818181818181816" + ], + [ + 1795102314.987, + "2.5275484598319817" + ], + [ + 1795102319.987, + "2.5090909090909093" + ], + [ + 1795102324.987, + "2.5274105860319653" + ], + [ + 1795102329.987, + "2.509090909090909" + ], + [ + 1795102334.987, + "2.5272727272727273" + ], + [ + 1795102339.987, + "2.509090909090909" + ], + [ + 1795102344.987, + "2.5272727272727273" + ], + [ + 1795102349.987, + "2.4909090909090907" + ], + [ + 1795102354.987, + "2.5090909090909093" + ], + [ + 1795102359.987, + "2.4909090909090907" + ], + [ + 1795102364.987, + "2.509090909090909" + ], + [ + 1795102369.987, + "2.4907732305510613" + ], + [ + 1795102374.987, + "2.509090909090909" + ], + [ + 1795102379.987, + "2.4909090909090907" + ], + [ + 1795102384.987, + "2.509090909090909" + ], + [ + 1795102389.987, + "2.4907732305510613" + ], + [ + 1795102394.987, + "2.509090909090909" + ], + [ + 1795102399.987, + "2.4909090909090907" + ], + [ + 1795102404.987, + "2.509090909090909" + ], + [ + 1795102409.987, + "2.4909090909090907" + ], + [ + 1795102414.987, + "2.509090909090909" + ], + [ + 1795102419.987, + "2.4909090909090907" + ], + [ + 1795102424.987, + "2.5092277760605124" + ], + [ + 1795102429.987, + "2.4909543809887453" + ], + [ + 1795102434.987, + "2.509090909090909" + ], + [ + 1795102439.987, + "2.4909090909090907" + ], + [ + 1795102444.987, + "2.509273401701942" + ], + [ + 1795102449.987, + "2.4909090909090907" + ], + [ + 1795102454.987, + "2.509090909090909" + ], + [ + 1795102459.987, + "2.4909090909090907" + ], + [ + 1795102464.987, + "2.509090909090909" + ], + [ + 1795102469.987, + "2.490773230551061" + ], + [ + 1795102474.987, + "2.509090909090909" + ], + [ + 1795102479.987, + "2.4909090909090907" + ], + [ + 1795102484.987, + "2.5090452900856346" + ], + [ + 1795102489.987, + "2.4909090909090907" + ], + [ + 1795102494.987, + "2.5091365297550867" + ], + [ + 1795102499.987, + "2.4908638024763188" + ], + [ + 1795102504.987, + "2.509090909090909" + ], + [ + 1795102509.987, + "2.4909543809887453" + ], + [ + 1795102514.987, + "2.509090909090909" + ], + [ + 1795102519.987, + "2.4909090909090907" + ], + [ + 1795102524.987, + "2.509273401701942" + ], + [ + 1795102529.987, + "2.4909090909090907" + ], + [ + 1795102534.987, + "2.509090909090909" + ], + [ + 1795102539.987, + "2.4909543809887458" + ], + [ + 1795102544.987, + "2.509090909090909" + ], + [ + 1795102549.987, + "2.4905015542911157" + ], + [ + 1795102554.987, + "2.509090909090909" + ], + [ + 1795102559.987, + "2.4907732305510613" + ], + [ + 1795102564.987, + "2.5090452900856346" + ], + [ + 1795102569.987, + "2.4906826652122533" + ], + [ + 1795102574.987, + "2.508954057051434" + ], + [ + 1795102579.987, + "2.490637385012544" + ], + [ + 1795102584.987, + "2.508862830651759" + ], + [ + 1795102589.987, + "2.4909543809887458" + ] + ] + } + ] + } + }, + "http_p95": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {}, + "values": [ + [ + 1795101974.987, + "NaN" + ], + [ + 1795101979.987, + "NaN" + ], + [ + 1795101984.987, + "0.03802910378684469" + ], + [ + 1795101989.987, + "0.03749999999999998" + ], + [ + 1795101994.987, + "0.036992187499999975" + ], + [ + 1795101999.987, + "0.03693181818181816" + ], + [ + 1795102004.987, + "0.03643939393939396" + ], + [ + 1795102009.987, + "0.03598484848484848" + ], + [ + 1795102014.987, + "0.03632142857142857" + ], + [ + 1795102019.987, + "0.035892857142857115" + ], + [ + 1795102024.987, + "0.035428571428571434" + ], + [ + 1795102029.987, + "0.03498249623478699" + ], + [ + 1795102034.987, + "0.034535714285714274" + ], + [ + 1795102039.987, + "0.02455952380952381" + ], + [ + 1795102044.987, + "0.024298387096774194" + ], + [ + 1795102049.987, + "0.024275423728813562" + ], + [ + 1795102054.987, + "0.024040983606557383" + ], + [ + 1795102059.987, + "0.02405327868852459" + ], + [ + 1795102064.987, + "0.024056451612903224" + ], + [ + 1795102069.987, + "0.023700892857142854" + ], + [ + 1795102074.987, + "0.02371052631578947" + ], + [ + 1795102079.987, + "0.02365277777777778" + ], + [ + 1795102084.987, + "0.023638888888888883" + ], + [ + 1795102089.987, + "0.023544999999999996" + ], + [ + 1795102094.987, + "0.02329807692307692" + ], + [ + 1795102099.987, + "0.023245" + ], + [ + 1795102104.987, + "0.023499999999999997" + ], + [ + 1795102109.987, + "0.027916666666666538" + ], + [ + 1795102114.987, + "0.02749999999999981" + ], + [ + 1795102119.987, + "0.027916666666666645" + ], + [ + 1795102124.987, + "0.02479" + ], + [ + 1795102129.987, + "0.027916666666666506" + ], + [ + 1795102134.987, + "0.02749999999999982" + ], + [ + 1795102139.987, + "0.027916666666666506" + ], + [ + 1795102144.987, + "0.02749999999999982" + ], + [ + 1795102149.987, + "0.027916666666666645" + ], + [ + 1795102154.987, + "0.02749999999999982" + ], + [ + 1795102159.987, + "0.02481603773584905" + ], + [ + 1795102164.987, + "0.02339090909090909" + ], + [ + 1795102169.987, + "0.023274999999999997" + ], + [ + 1795102174.987, + "2.4875" + ], + [ + 1795102179.987, + "3.891666666666665" + ], + [ + 1795102184.987, + "4.282142857142857" + ], + [ + 1795102189.987, + "4.475" + ], + [ + 1795102194.987, + "4.58125" + ], + [ + 1795102199.987, + "4.656034482758621" + ], + [ + 1795102204.987, + "4.704411764705882" + ], + [ + 1795102209.987, + "4.744230769230769" + ], + [ + 1795102214.987, + "4.771590909090909" + ], + [ + 1795102219.987, + "4.796428571428571" + ], + [ + 1795102224.987, + "4.811111111111111" + ], + [ + 1795102229.987, + "4.813181818181818" + ], + [ + 1795102234.987, + "4.8118181818181816" + ], + [ + 1795102239.987, + "4.813181818181818" + ], + [ + 1795102244.987, + "4.8118181818181816" + ], + [ + 1795102249.987, + "4.8131818181818184" + ], + [ + 1795102254.987, + "4.811818181818182" + ], + [ + 1795102259.987, + "4.804166666666667" + ], + [ + 1795102264.987, + "4.782653061224489" + ], + [ + 1795102269.987, + "4.759659090909091" + ], + [ + 1795102274.987, + "4.726923076923077" + ], + [ + 1795102279.987, + "4.688970588235294" + ], + [ + 1795102284.987, + "4.632758620689654" + ], + [ + 1795102289.987, + "4.559374999999999" + ], + [ + 1795102294.987, + "4.435526315789473" + ], + [ + 1795102299.987, + "4.239285714285713" + ], + [ + 1795102304.987, + "3.808333333333332" + ], + [ + 1795102309.987, + "2.3374999999999906" + ], + [ + 1795102314.987, + "0.023982876712328766" + ], + [ + 1795102319.987, + "0.023679104477611946" + ], + [ + 1795102324.987, + "0.02360546875" + ], + [ + 1795102329.987, + "0.023499999999999997" + ], + [ + 1795102334.987, + "0.023434210526315787" + ], + [ + 1795102339.987, + "0.023419642857142854" + ], + [ + 1795102344.987, + "0.023347222222222228" + ], + [ + 1795102349.987, + "0.023209183673469386" + ], + [ + 1795102354.987, + "0.023558823529411767" + ], + [ + 1795102359.987, + "0.023132978723404255" + ], + [ + 1795102364.987, + "0.02311702127659574" + ], + [ + 1795102369.987, + "0.023049999999999998" + ], + [ + 1795102374.987, + "0.02303333333333333" + ], + [ + 1795102379.987, + "0.023346590909090904" + ], + [ + 1795102384.987, + "0.023402173913043472" + ], + [ + 1795102389.987, + "0.02338333333333333" + ], + [ + 1795102394.987, + "0.023402173913043472" + ], + [ + 1795102399.987, + "0.02341847826086956" + ], + [ + 1795102404.987, + "0.023468749999999997" + ], + [ + 1795102409.987, + "0.023132978723404255" + ], + [ + 1795102414.987, + "0.023264705882352937" + ], + [ + 1795102419.987, + "0.023209183673469386" + ], + [ + 1795102424.987, + "0.02326470588235294" + ], + [ + 1795102429.987, + "0.023545" + ], + [ + 1795102434.987, + "0.023264705882352937" + ], + [ + 1795102439.987, + "0.02341847826086956" + ], + [ + 1795102444.987, + "0.023468749999999997" + ], + [ + 1795102449.987, + "0.023452127659574464" + ], + [ + 1795102454.987, + "0.023436170212765953" + ], + [ + 1795102459.987, + "0.023452127659574464" + ], + [ + 1795102464.987, + "0.023468749999999997" + ], + [ + 1795102469.987, + "0.023484375" + ], + [ + 1795102474.987, + "0.023499999999999997" + ], + [ + 1795102479.987, + "0.023484374999999995" + ], + [ + 1795102484.987, + "0.023076086956521732" + ], + [ + 1795102489.987, + "0.02285975609756097" + ], + [ + 1795102494.987, + "0.022346153846153842" + ], + [ + 1795102499.987, + "0.022365384615384613" + ], + [ + 1795102504.987, + "0.022346153846153842" + ], + [ + 1795102509.987, + "0.022365384615384613" + ], + [ + 1795102514.987, + "0.022276315789473682" + ], + [ + 1795102519.987, + "0.022296052631578946" + ], + [ + 1795102524.987, + "0.022125" + ], + [ + 1795102529.987, + "0.022064285714285713" + ], + [ + 1795102534.987, + "0.02204285714285714" + ], + [ + 1795102539.987, + "0.022296052631578946" + ], + [ + 1795102544.987, + "0.02264772727272727" + ], + [ + 1795102549.987, + "0.022945" + ], + [ + 1795102554.987, + "0.022887755102040813" + ], + [ + 1795102559.987, + "0.022859374999999998" + ], + [ + 1795102564.987, + "0.022887755102040816" + ], + [ + 1795102569.987, + "0.022903061224489794" + ], + [ + 1795102574.987, + "0.02323" + ], + [ + 1795102579.987, + "0.023544999999999996" + ], + [ + 1795102584.987, + "0.02358653846153846" + ], + [ + 1795102589.987, + "0.023627358490566044" + ] + ] + } + ] + } + }, + "queue_depth": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {}, + "values": [ + [ + 1795101974.987, + "0" + ], + [ + 1795101979.987, + "0" + ], + [ + 1795101984.987, + "1" + ], + [ + 1795101989.987, + "1" + ], + [ + 1795101994.987, + "0" + ], + [ + 1795101999.987, + "1" + ], + [ + 1795102004.987, + "0" + ], + [ + 1795102009.987, + "0" + ], + [ + 1795102014.987, + "0" + ], + [ + 1795102019.987, + "0" + ], + [ + 1795102024.987, + "1" + ], + [ + 1795102029.987, + "0" + ], + [ + 1795102034.987, + "1" + ], + [ + 1795102039.987, + "0" + ], + [ + 1795102044.987, + "0" + ], + [ + 1795102049.987, + "0" + ], + [ + 1795102054.987, + "0" + ], + [ + 1795102059.987, + "0" + ], + [ + 1795102064.987, + "0" + ], + [ + 1795102069.987, + "1" + ], + [ + 1795102074.987, + "0" + ], + [ + 1795102079.987, + "1" + ], + [ + 1795102084.987, + "0" + ], + [ + 1795102089.987, + "0" + ], + [ + 1795102094.987, + "0" + ], + [ + 1795102099.987, + "0" + ], + [ + 1795102104.987, + "1" + ], + [ + 1795102109.987, + "0" + ], + [ + 1795102114.987, + "1" + ], + [ + 1795102119.987, + "0" + ], + [ + 1795102124.987, + "0" + ], + [ + 1795102129.987, + "0" + ], + [ + 1795102134.987, + "0" + ], + [ + 1795102139.987, + "0" + ], + [ + 1795102144.987, + "0" + ], + [ + 1795102149.987, + "1" + ], + [ + 1795102154.987, + "0" + ], + [ + 1795102159.987, + "1" + ], + [ + 1795102164.987, + "0" + ], + [ + 1795102169.987, + "0" + ], + [ + 1795102174.987, + "0" + ], + [ + 1795102179.987, + "0" + ], + [ + 1795102184.987, + "1" + ], + [ + 1795102189.987, + "1" + ], + [ + 1795102194.987, + "0" + ], + [ + 1795102199.987, + "1" + ], + [ + 1795102204.987, + "0" + ], + [ + 1795102209.987, + "0" + ], + [ + 1795102214.987, + "0" + ], + [ + 1795102219.987, + "0" + ], + [ + 1795102224.987, + "0" + ], + [ + 1795102229.987, + "0" + ], + [ + 1795102234.987, + "1" + ], + [ + 1795102239.987, + "0" + ], + [ + 1795102244.987, + "1" + ], + [ + 1795102249.987, + "0" + ], + [ + 1795102254.987, + "0" + ], + [ + 1795102259.987, + "0" + ], + [ + 1795102264.987, + "0" + ], + [ + 1795102269.987, + "1" + ], + [ + 1795102274.987, + "0" + ], + [ + 1795102279.987, + "1" + ], + [ + 1795102284.987, + "0" + ], + [ + 1795102289.987, + "0" + ], + [ + 1795102294.987, + "0" + ], + [ + 1795102299.987, + "0" + ], + [ + 1795102304.987, + "0" + ], + [ + 1795102309.987, + "0" + ], + [ + 1795102314.987, + "1" + ], + [ + 1795102319.987, + "0" + ], + [ + 1795102324.987, + "1" + ], + [ + 1795102329.987, + "0" + ], + [ + 1795102334.987, + "1" + ], + [ + 1795102339.987, + "0" + ], + [ + 1795102344.987, + "0" + ], + [ + 1795102349.987, + "0" + ], + [ + 1795102354.987, + "0" + ], + [ + 1795102359.987, + "1" + ], + [ + 1795102364.987, + "1" + ], + [ + 1795102369.987, + "4" + ], + [ + 1795102374.987, + "6" + ], + [ + 1795102379.987, + "9" + ], + [ + 1795102384.987, + "11" + ], + [ + 1795102389.987, + "14" + ], + [ + 1795102394.987, + "16" + ], + [ + 1795102399.987, + "19" + ], + [ + 1795102404.987, + "22" + ], + [ + 1795102409.987, + "24" + ], + [ + 1795102414.987, + "27" + ], + [ + 1795102419.987, + "27" + ], + [ + 1795102424.987, + "29" + ], + [ + 1795102429.987, + "32" + ], + [ + 1795102434.987, + "34" + ], + [ + 1795102439.987, + "37" + ], + [ + 1795102444.987, + "39" + ], + [ + 1795102449.987, + "42" + ], + [ + 1795102454.987, + "45" + ], + [ + 1795102459.987, + "47" + ], + [ + 1795102464.987, + "50" + ], + [ + 1795102469.987, + "52" + ], + [ + 1795102474.987, + "55" + ], + [ + 1795102479.987, + "57" + ], + [ + 1795102484.987, + "60" + ], + [ + 1795102489.987, + "51" + ], + [ + 1795102494.987, + "25" + ], + [ + 1795102499.987, + "1" + ], + [ + 1795102504.987, + "0" + ], + [ + 1795102509.987, + "1" + ], + [ + 1795102514.987, + "0" + ], + [ + 1795102519.987, + "0" + ], + [ + 1795102524.987, + "0" + ], + [ + 1795102529.987, + "0" + ], + [ + 1795102534.987, + "0" + ], + [ + 1795102539.987, + "0" + ], + [ + 1795102544.987, + "1" + ], + [ + 1795102549.987, + "0" + ], + [ + 1795102554.987, + "1" + ], + [ + 1795102559.987, + "0" + ], + [ + 1795102564.987, + "0" + ], + [ + 1795102569.987, + "0" + ], + [ + 1795102574.987, + "0" + ], + [ + 1795102579.987, + "0" + ], + [ + 1795102584.987, + "0" + ], + [ + 1795102589.987, + "1" + ] + ] + } + ] + } + }, + "queue_age": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {}, + "values": [ + [ + 1795101974.987, + "0" + ], + [ + 1795101979.987, + "0" + ], + [ + 1795101984.987, + "0.479455" + ], + [ + 1795101989.987, + "0.191489" + ], + [ + 1795101994.987, + "0" + ], + [ + 1795101999.987, + "0.43069" + ], + [ + 1795102004.987, + "0" + ], + [ + 1795102009.987, + "0" + ], + [ + 1795102014.987, + "0" + ], + [ + 1795102019.987, + "0" + ], + [ + 1795102024.987, + "0.051709" + ], + [ + 1795102029.987, + "0" + ], + [ + 1795102034.987, + "0.286549" + ], + [ + 1795102039.987, + "0" + ], + [ + 1795102044.987, + "0" + ], + [ + 1795102049.987, + "0" + ], + [ + 1795102054.987, + "0" + ], + [ + 1795102059.987, + "0" + ], + [ + 1795102064.987, + "0" + ], + [ + 1795102069.987, + "0.148366" + ], + [ + 1795102074.987, + "0" + ], + [ + 1795102079.987, + "0.427642" + ], + [ + 1795102084.987, + "0" + ], + [ + 1795102089.987, + "0" + ], + [ + 1795102094.987, + "0" + ], + [ + 1795102099.987, + "0" + ], + [ + 1795102104.987, + "0.090217" + ], + [ + 1795102109.987, + "0" + ], + [ + 1795102114.987, + "0.30598" + ], + [ + 1795102119.987, + "0" + ], + [ + 1795102124.987, + "0" + ], + [ + 1795102129.987, + "0" + ], + [ + 1795102134.987, + "0" + ], + [ + 1795102139.987, + "0" + ], + [ + 1795102144.987, + "0" + ], + [ + 1795102149.987, + "0.242537" + ], + [ + 1795102154.987, + "0" + ], + [ + 1795102159.987, + "0.473372" + ], + [ + 1795102164.987, + "0" + ], + [ + 1795102169.987, + "0" + ], + [ + 1795102174.987, + "0" + ], + [ + 1795102179.987, + "0" + ], + [ + 1795102184.987, + "0.116103" + ], + [ + 1795102189.987, + "0.116103" + ], + [ + 1795102194.987, + "0" + ], + [ + 1795102199.987, + "0.372153" + ], + [ + 1795102204.987, + "0" + ], + [ + 1795102209.987, + "0" + ], + [ + 1795102214.987, + "0" + ], + [ + 1795102219.987, + "0" + ], + [ + 1795102224.987, + "0" + ], + [ + 1795102229.987, + "0" + ], + [ + 1795102234.987, + "0.236601" + ], + [ + 1795102239.987, + "0" + ], + [ + 1795102244.987, + "0.510616" + ], + [ + 1795102249.987, + "0" + ], + [ + 1795102254.987, + "0" + ], + [ + 1795102259.987, + "0" + ], + [ + 1795102264.987, + "0" + ], + [ + 1795102269.987, + "0.076095" + ], + [ + 1795102274.987, + "0" + ], + [ + 1795102279.987, + "0.313001" + ], + [ + 1795102284.987, + "0" + ], + [ + 1795102289.987, + "0" + ], + [ + 1795102294.987, + "0" + ], + [ + 1795102299.987, + "0" + ], + [ + 1795102304.987, + "0" + ], + [ + 1795102309.987, + "0" + ], + [ + 1795102314.987, + "0.083168" + ], + [ + 1795102319.987, + "0" + ], + [ + 1795102324.987, + "0.271703" + ], + [ + 1795102329.987, + "0" + ], + [ + 1795102334.987, + "0.493807" + ], + [ + 1795102339.987, + "0" + ], + [ + 1795102344.987, + "0" + ], + [ + 1795102349.987, + "0" + ], + [ + 1795102354.987, + "0" + ], + [ + 1795102359.987, + "0.03898" + ], + [ + 1795102364.987, + "1.14508" + ], + [ + 1795102369.987, + "6.249117" + ], + [ + 1795102374.987, + "11.382691" + ], + [ + 1795102379.987, + "16.507015" + ], + [ + 1795102384.987, + "21.648112" + ], + [ + 1795102389.987, + "26.774682" + ], + [ + 1795102394.987, + "31.875267" + ], + [ + 1795102399.987, + "36.998759" + ], + [ + 1795102404.987, + "42.124386" + ], + [ + 1795102409.987, + "47.243046" + ], + [ + 1795102414.987, + "52.368544" + ], + [ + 1795102419.987, + "52.368544" + ], + [ + 1795102424.987, + "57.523482" + ], + [ + 1795102429.987, + "62.648557" + ], + [ + 1795102434.987, + "67.747257" + ], + [ + 1795102439.987, + "72.856874" + ], + [ + 1795102444.987, + "77.986024" + ], + [ + 1795102449.987, + "83.109209" + ], + [ + 1795102454.987, + "88.220407" + ], + [ + 1795102459.987, + "93.351768" + ], + [ + 1795102464.987, + "98.473515" + ], + [ + 1795102469.987, + "103.563256" + ], + [ + 1795102474.987, + "108.703388" + ], + [ + 1795102479.987, + "113.829774" + ], + [ + 1795102484.987, + "118.941931" + ], + [ + 1795102489.987, + "100.000807" + ], + [ + 1795102494.987, + "49.058342" + ], + [ + 1795102499.987, + "0.110804" + ], + [ + 1795102504.987, + "0" + ], + [ + 1795102509.987, + "0.3242" + ], + [ + 1795102514.987, + "0" + ], + [ + 1795102519.987, + "0" + ], + [ + 1795102524.987, + "0" + ], + [ + 1795102529.987, + "0" + ], + [ + 1795102534.987, + "0" + ], + [ + 1795102539.987, + "0" + ], + [ + 1795102544.987, + "0.100515" + ], + [ + 1795102549.987, + "0" + ], + [ + 1795102554.987, + "0.396407" + ], + [ + 1795102559.987, + "0" + ], + [ + 1795102564.987, + "0" + ], + [ + 1795102569.987, + "0" + ], + [ + 1795102574.987, + "0" + ], + [ + 1795102579.987, + "0" + ], + [ + 1795102584.987, + "0" + ], + [ + 1795102589.987, + "0.315399" + ] + ] + } + ] + } + }, + "db_pool": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "state": "used" + }, + "values": [ + [ + 1795101974.987, + "9" + ], + [ + 1795101979.987, + "9" + ], + [ + 1795101984.987, + "9" + ], + [ + 1795101989.987, + "9" + ], + [ + 1795101994.987, + "0" + ], + [ + 1795101999.987, + "0" + ], + [ + 1795102004.987, + "0" + ], + [ + 1795102009.987, + "0" + ], + [ + 1795102014.987, + "0" + ], + [ + 1795102019.987, + "0" + ], + [ + 1795102024.987, + "0" + ], + [ + 1795102029.987, + "0" + ], + [ + 1795102034.987, + "0" + ], + [ + 1795102039.987, + "0" + ], + [ + 1795102044.987, + "1" + ], + [ + 1795102049.987, + "0" + ], + [ + 1795102054.987, + "0" + ], + [ + 1795102059.987, + "0" + ], + [ + 1795102064.987, + "0" + ], + [ + 1795102069.987, + "0" + ], + [ + 1795102074.987, + "0" + ], + [ + 1795102079.987, + "0" + ], + [ + 1795102084.987, + "0" + ], + [ + 1795102089.987, + "0" + ], + [ + 1795102094.987, + "0" + ], + [ + 1795102099.987, + "0" + ], + [ + 1795102104.987, + "0" + ], + [ + 1795102109.987, + "0" + ], + [ + 1795102114.987, + "0" + ], + [ + 1795102119.987, + "0" + ], + [ + 1795102124.987, + "0" + ], + [ + 1795102129.987, + "0" + ], + [ + 1795102134.987, + "0" + ], + [ + 1795102139.987, + "1" + ], + [ + 1795102144.987, + "0" + ], + [ + 1795102149.987, + "0" + ], + [ + 1795102154.987, + "0" + ], + [ + 1795102159.987, + "0" + ], + [ + 1795102164.987, + "0" + ], + [ + 1795102169.987, + "2" + ], + [ + 1795102174.987, + "4" + ], + [ + 1795102179.987, + "4" + ], + [ + 1795102184.987, + "4" + ], + [ + 1795102189.987, + "4" + ], + [ + 1795102194.987, + "4" + ], + [ + 1795102199.987, + "4" + ], + [ + 1795102204.987, + "4" + ], + [ + 1795102209.987, + "4" + ], + [ + 1795102214.987, + "4" + ], + [ + 1795102219.987, + "4" + ], + [ + 1795102224.987, + "4" + ], + [ + 1795102229.987, + "5" + ], + [ + 1795102234.987, + "4" + ], + [ + 1795102239.987, + "4" + ], + [ + 1795102244.987, + "4" + ], + [ + 1795102249.987, + "5" + ], + [ + 1795102254.987, + "4" + ], + [ + 1795102259.987, + "0" + ], + [ + 1795102264.987, + "0" + ], + [ + 1795102269.987, + "0" + ], + [ + 1795102274.987, + "0" + ], + [ + 1795102279.987, + "0" + ], + [ + 1795102284.987, + "0" + ], + [ + 1795102289.987, + "0" + ], + [ + 1795102294.987, + "0" + ], + [ + 1795102299.987, + "0" + ], + [ + 1795102304.987, + "0" + ], + [ + 1795102309.987, + "0" + ], + [ + 1795102314.987, + "0" + ], + [ + 1795102319.987, + "0" + ], + [ + 1795102324.987, + "0" + ], + [ + 1795102329.987, + "0" + ], + [ + 1795102334.987, + "0" + ], + [ + 1795102339.987, + "0" + ], + [ + 1795102344.987, + "0" + ], + [ + 1795102349.987, + "0" + ], + [ + 1795102354.987, + "0" + ], + [ + 1795102359.987, + "0" + ], + [ + 1795102364.987, + "0" + ], + [ + 1795102369.987, + "0" + ], + [ + 1795102374.987, + "0" + ], + [ + 1795102379.987, + "0" + ], + [ + 1795102384.987, + "0" + ], + [ + 1795102389.987, + "0" + ], + [ + 1795102394.987, + "0" + ], + [ + 1795102399.987, + "0" + ], + [ + 1795102404.987, + "0" + ], + [ + 1795102409.987, + "0" + ], + [ + 1795102414.987, + "0" + ], + [ + 1795102419.987, + "0" + ], + [ + 1795102424.987, + "2" + ], + [ + 1795102429.987, + "0" + ], + [ + 1795102434.987, + "0" + ], + [ + 1795102439.987, + "0" + ], + [ + 1795102444.987, + "0" + ], + [ + 1795102449.987, + "0" + ], + [ + 1795102454.987, + "0" + ], + [ + 1795102459.987, + "0" + ], + [ + 1795102464.987, + "0" + ], + [ + 1795102469.987, + "0" + ], + [ + 1795102474.987, + "0" + ], + [ + 1795102479.987, + "0" + ], + [ + 1795102484.987, + "0" + ], + [ + 1795102489.987, + "0" + ], + [ + 1795102494.987, + "0" + ], + [ + 1795102499.987, + "0" + ], + [ + 1795102504.987, + "0" + ], + [ + 1795102509.987, + "0" + ], + [ + 1795102514.987, + "0" + ], + [ + 1795102519.987, + "0" + ], + [ + 1795102524.987, + "0" + ], + [ + 1795102529.987, + "0" + ], + [ + 1795102534.987, + "0" + ], + [ + 1795102539.987, + "0" + ], + [ + 1795102544.987, + "0" + ], + [ + 1795102549.987, + "0" + ], + [ + 1795102554.987, + "0" + ], + [ + 1795102559.987, + "0" + ], + [ + 1795102564.987, + "0" + ], + [ + 1795102569.987, + "0" + ], + [ + 1795102574.987, + "0" + ], + [ + 1795102579.987, + "0" + ], + [ + 1795102584.987, + "0" + ], + [ + 1795102589.987, + "0" + ] + ] + }, + { + "metric": { + "state": "waiting" + }, + "values": [ + [ + 1795101974.987, + "0" + ], + [ + 1795101979.987, + "0" + ], + [ + 1795101984.987, + "0" + ], + [ + 1795101989.987, + "0" + ], + [ + 1795101994.987, + "0" + ], + [ + 1795101999.987, + "0" + ], + [ + 1795102004.987, + "0" + ], + [ + 1795102009.987, + "0" + ], + [ + 1795102014.987, + "0" + ], + [ + 1795102019.987, + "0" + ], + [ + 1795102024.987, + "0" + ], + [ + 1795102029.987, + "0" + ], + [ + 1795102034.987, + "0" + ], + [ + 1795102039.987, + "0" + ], + [ + 1795102044.987, + "0" + ], + [ + 1795102049.987, + "0" + ], + [ + 1795102054.987, + "0" + ], + [ + 1795102059.987, + "0" + ], + [ + 1795102064.987, + "0" + ], + [ + 1795102069.987, + "0" + ], + [ + 1795102074.987, + "0" + ], + [ + 1795102079.987, + "0" + ], + [ + 1795102084.987, + "0" + ], + [ + 1795102089.987, + "0" + ], + [ + 1795102094.987, + "0" + ], + [ + 1795102099.987, + "0" + ], + [ + 1795102104.987, + "0" + ], + [ + 1795102109.987, + "0" + ], + [ + 1795102114.987, + "0" + ], + [ + 1795102119.987, + "0" + ], + [ + 1795102124.987, + "0" + ], + [ + 1795102129.987, + "0" + ], + [ + 1795102134.987, + "0" + ], + [ + 1795102139.987, + "0" + ], + [ + 1795102144.987, + "0" + ], + [ + 1795102149.987, + "0" + ], + [ + 1795102154.987, + "0" + ], + [ + 1795102159.987, + "0" + ], + [ + 1795102164.987, + "0" + ], + [ + 1795102169.987, + "0" + ], + [ + 1795102174.987, + "0" + ], + [ + 1795102179.987, + "0" + ], + [ + 1795102184.987, + "0" + ], + [ + 1795102189.987, + "0" + ], + [ + 1795102194.987, + "0" + ], + [ + 1795102199.987, + "0" + ], + [ + 1795102204.987, + "0" + ], + [ + 1795102209.987, + "0" + ], + [ + 1795102214.987, + "0" + ], + [ + 1795102219.987, + "0" + ], + [ + 1795102224.987, + "0" + ], + [ + 1795102229.987, + "0" + ], + [ + 1795102234.987, + "0" + ], + [ + 1795102239.987, + "0" + ], + [ + 1795102244.987, + "0" + ], + [ + 1795102249.987, + "0" + ], + [ + 1795102254.987, + "0" + ], + [ + 1795102259.987, + "0" + ], + [ + 1795102264.987, + "0" + ], + [ + 1795102269.987, + "0" + ], + [ + 1795102274.987, + "0" + ], + [ + 1795102279.987, + "0" + ], + [ + 1795102284.987, + "0" + ], + [ + 1795102289.987, + "0" + ], + [ + 1795102294.987, + "0" + ], + [ + 1795102299.987, + "0" + ], + [ + 1795102304.987, + "0" + ], + [ + 1795102309.987, + "0" + ], + [ + 1795102314.987, + "0" + ], + [ + 1795102319.987, + "0" + ], + [ + 1795102324.987, + "0" + ], + [ + 1795102329.987, + "0" + ], + [ + 1795102334.987, + "0" + ], + [ + 1795102339.987, + "0" + ], + [ + 1795102344.987, + "0" + ], + [ + 1795102349.987, + "0" + ], + [ + 1795102354.987, + "0" + ], + [ + 1795102359.987, + "0" + ], + [ + 1795102364.987, + "0" + ], + [ + 1795102369.987, + "0" + ], + [ + 1795102374.987, + "0" + ], + [ + 1795102379.987, + "0" + ], + [ + 1795102384.987, + "0" + ], + [ + 1795102389.987, + "0" + ], + [ + 1795102394.987, + "0" + ], + [ + 1795102399.987, + "0" + ], + [ + 1795102404.987, + "0" + ], + [ + 1795102409.987, + "0" + ], + [ + 1795102414.987, + "0" + ], + [ + 1795102419.987, + "0" + ], + [ + 1795102424.987, + "0" + ], + [ + 1795102429.987, + "0" + ], + [ + 1795102434.987, + "0" + ], + [ + 1795102439.987, + "0" + ], + [ + 1795102444.987, + "0" + ], + [ + 1795102449.987, + "0" + ], + [ + 1795102454.987, + "0" + ], + [ + 1795102459.987, + "0" + ], + [ + 1795102464.987, + "0" + ], + [ + 1795102469.987, + "0" + ], + [ + 1795102474.987, + "0" + ], + [ + 1795102479.987, + "0" + ], + [ + 1795102484.987, + "0" + ], + [ + 1795102489.987, + "0" + ], + [ + 1795102494.987, + "0" + ], + [ + 1795102499.987, + "0" + ], + [ + 1795102504.987, + "0" + ], + [ + 1795102509.987, + "0" + ], + [ + 1795102514.987, + "0" + ], + [ + 1795102519.987, + "0" + ], + [ + 1795102524.987, + "0" + ], + [ + 1795102529.987, + "0" + ], + [ + 1795102534.987, + "0" + ], + [ + 1795102539.987, + "0" + ], + [ + 1795102544.987, + "0" + ], + [ + 1795102549.987, + "0" + ], + [ + 1795102554.987, + "0" + ], + [ + 1795102559.987, + "0" + ], + [ + 1795102564.987, + "0" + ], + [ + 1795102569.987, + "0" + ], + [ + 1795102574.987, + "0" + ], + [ + 1795102579.987, + "0" + ], + [ + 1795102584.987, + "0" + ], + [ + 1795102589.987, + "0" + ] + ] + } + ] + } + }, + "worker_busy": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {}, + "values": [ + [ + 1795101974.987, + "0" + ], + [ + 1795101979.987, + "1" + ], + [ + 1795101984.987, + "1" + ], + [ + 1795101989.987, + "0" + ], + [ + 1795101994.987, + "1" + ], + [ + 1795101999.987, + "0" + ], + [ + 1795102004.987, + "1" + ], + [ + 1795102009.987, + "0" + ], + [ + 1795102014.987, + "1" + ], + [ + 1795102019.987, + "0" + ], + [ + 1795102024.987, + "1" + ], + [ + 1795102029.987, + "0" + ], + [ + 1795102034.987, + "1" + ], + [ + 1795102039.987, + "0" + ], + [ + 1795102044.987, + "1" + ], + [ + 1795102049.987, + "0" + ], + [ + 1795102054.987, + "0" + ], + [ + 1795102059.987, + "0" + ], + [ + 1795102064.987, + "0" + ], + [ + 1795102069.987, + "0" + ], + [ + 1795102074.987, + "0" + ], + [ + 1795102079.987, + "0" + ], + [ + 1795102084.987, + "0" + ], + [ + 1795102089.987, + "0" + ], + [ + 1795102094.987, + "0" + ], + [ + 1795102099.987, + "0" + ], + [ + 1795102104.987, + "0" + ], + [ + 1795102109.987, + "0" + ], + [ + 1795102114.987, + "0" + ], + [ + 1795102119.987, + "0" + ], + [ + 1795102124.987, + "0" + ], + [ + 1795102129.987, + "0" + ], + [ + 1795102134.987, + "0" + ], + [ + 1795102139.987, + "0" + ], + [ + 1795102144.987, + "0" + ], + [ + 1795102149.987, + "0" + ], + [ + 1795102154.987, + "0" + ], + [ + 1795102159.987, + "0" + ], + [ + 1795102164.987, + "0" + ], + [ + 1795102169.987, + "0" + ], + [ + 1795102174.987, + "0" + ], + [ + 1795102179.987, + "0" + ], + [ + 1795102184.987, + "0" + ], + [ + 1795102189.987, + "0" + ], + [ + 1795102194.987, + "0" + ], + [ + 1795102199.987, + "0" + ], + [ + 1795102204.987, + "0" + ], + [ + 1795102209.987, + "0" + ], + [ + 1795102214.987, + "0" + ], + [ + 1795102219.987, + "0" + ], + [ + 1795102224.987, + "0" + ], + [ + 1795102229.987, + "0" + ], + [ + 1795102234.987, + "0" + ], + [ + 1795102239.987, + "0" + ], + [ + 1795102244.987, + "0" + ], + [ + 1795102249.987, + "0" + ], + [ + 1795102254.987, + "0" + ], + [ + 1795102259.987, + "0" + ], + [ + 1795102264.987, + "0" + ], + [ + 1795102269.987, + "0" + ], + [ + 1795102274.987, + "0" + ], + [ + 1795102279.987, + "0" + ], + [ + 1795102284.987, + "0" + ], + [ + 1795102289.987, + "0" + ], + [ + 1795102294.987, + "0" + ], + [ + 1795102299.987, + "0" + ], + [ + 1795102304.987, + "0" + ], + [ + 1795102309.987, + "0" + ], + [ + 1795102314.987, + "0" + ], + [ + 1795102319.987, + "0" + ], + [ + 1795102324.987, + "0" + ], + [ + 1795102329.987, + "0" + ], + [ + 1795102334.987, + "0" + ], + [ + 1795102339.987, + "0" + ], + [ + 1795102344.987, + "0" + ], + [ + 1795102349.987, + "0" + ], + [ + 1795102354.987, + "0" + ], + [ + 1795102359.987, + "0" + ], + [ + 1795102364.987, + "0" + ], + [ + 1795102369.987, + "0" + ], + [ + 1795102374.987, + "0" + ], + [ + 1795102379.987, + "0" + ], + [ + 1795102384.987, + "0" + ], + [ + 1795102489.987, + "2" + ], + [ + 1795102494.987, + "2" + ], + [ + 1795102499.987, + "0" + ], + [ + 1795102504.987, + "0" + ], + [ + 1795102509.987, + "0" + ], + [ + 1795102514.987, + "0" + ], + [ + 1795102519.987, + "0" + ], + [ + 1795102524.987, + "0" + ], + [ + 1795102529.987, + "0" + ], + [ + 1795102534.987, + "0" + ], + [ + 1795102539.987, + "0" + ], + [ + 1795102544.987, + "0" + ], + [ + 1795102549.987, + "0" + ], + [ + 1795102554.987, + "0" + ], + [ + 1795102559.987, + "0" + ], + [ + 1795102564.987, + "0" + ], + [ + 1795102569.987, + "0" + ], + [ + 1795102574.987, + "0" + ], + [ + 1795102579.987, + "0" + ], + [ + 1795102584.987, + "0" + ], + [ + 1795102589.987, + "0" + ] + ] + } + ] + } + }, + "worker_capacity": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {}, + "values": [ + [ + 1795101974.987, + "4" + ], + [ + 1795101979.987, + "4" + ], + [ + 1795101984.987, + "4" + ], + [ + 1795101989.987, + "4" + ], + [ + 1795101994.987, + "2" + ], + [ + 1795101999.987, + "2" + ], + [ + 1795102004.987, + "2" + ], + [ + 1795102009.987, + "2" + ], + [ + 1795102014.987, + "2" + ], + [ + 1795102019.987, + "2" + ], + [ + 1795102024.987, + "2" + ], + [ + 1795102029.987, + "2" + ], + [ + 1795102034.987, + "2" + ], + [ + 1795102039.987, + "2" + ], + [ + 1795102044.987, + "2" + ], + [ + 1795102049.987, + "2" + ], + [ + 1795102054.987, + "2" + ], + [ + 1795102059.987, + "2" + ], + [ + 1795102064.987, + "2" + ], + [ + 1795102069.987, + "2" + ], + [ + 1795102074.987, + "2" + ], + [ + 1795102079.987, + "2" + ], + [ + 1795102084.987, + "2" + ], + [ + 1795102089.987, + "2" + ], + [ + 1795102094.987, + "2" + ], + [ + 1795102099.987, + "2" + ], + [ + 1795102104.987, + "2" + ], + [ + 1795102109.987, + "2" + ], + [ + 1795102114.987, + "2" + ], + [ + 1795102119.987, + "2" + ], + [ + 1795102124.987, + "2" + ], + [ + 1795102129.987, + "2" + ], + [ + 1795102134.987, + "2" + ], + [ + 1795102139.987, + "2" + ], + [ + 1795102144.987, + "2" + ], + [ + 1795102149.987, + "2" + ], + [ + 1795102154.987, + "2" + ], + [ + 1795102159.987, + "2" + ], + [ + 1795102164.987, + "2" + ], + [ + 1795102169.987, + "2" + ], + [ + 1795102174.987, + "2" + ], + [ + 1795102179.987, + "2" + ], + [ + 1795102184.987, + "2" + ], + [ + 1795102189.987, + "2" + ], + [ + 1795102194.987, + "2" + ], + [ + 1795102199.987, + "2" + ], + [ + 1795102204.987, + "2" + ], + [ + 1795102209.987, + "2" + ], + [ + 1795102214.987, + "2" + ], + [ + 1795102219.987, + "2" + ], + [ + 1795102224.987, + "2" + ], + [ + 1795102229.987, + "2" + ], + [ + 1795102234.987, + "2" + ], + [ + 1795102239.987, + "2" + ], + [ + 1795102244.987, + "2" + ], + [ + 1795102249.987, + "2" + ], + [ + 1795102254.987, + "2" + ], + [ + 1795102259.987, + "2" + ], + [ + 1795102264.987, + "2" + ], + [ + 1795102269.987, + "2" + ], + [ + 1795102274.987, + "2" + ], + [ + 1795102279.987, + "2" + ], + [ + 1795102284.987, + "2" + ], + [ + 1795102289.987, + "2" + ], + [ + 1795102294.987, + "2" + ], + [ + 1795102299.987, + "2" + ], + [ + 1795102304.987, + "2" + ], + [ + 1795102309.987, + "2" + ], + [ + 1795102314.987, + "2" + ], + [ + 1795102319.987, + "2" + ], + [ + 1795102324.987, + "2" + ], + [ + 1795102329.987, + "2" + ], + [ + 1795102334.987, + "2" + ], + [ + 1795102339.987, + "2" + ], + [ + 1795102344.987, + "2" + ], + [ + 1795102349.987, + "2" + ], + [ + 1795102354.987, + "2" + ], + [ + 1795102359.987, + "2" + ], + [ + 1795102364.987, + "2" + ], + [ + 1795102369.987, + "2" + ], + [ + 1795102374.987, + "2" + ], + [ + 1795102379.987, + "2" + ], + [ + 1795102384.987, + "2" + ], + [ + 1795102489.987, + "2" + ], + [ + 1795102494.987, + "2" + ], + [ + 1795102499.987, + "2" + ], + [ + 1795102504.987, + "2" + ], + [ + 1795102509.987, + "2" + ], + [ + 1795102514.987, + "2" + ], + [ + 1795102519.987, + "2" + ], + [ + 1795102524.987, + "2" + ], + [ + 1795102529.987, + "2" + ], + [ + 1795102534.987, + "2" + ], + [ + 1795102539.987, + "2" + ], + [ + 1795102544.987, + "2" + ], + [ + 1795102549.987, + "2" + ], + [ + 1795102554.987, + "2" + ], + [ + 1795102559.987, + "2" + ], + [ + 1795102564.987, + "2" + ], + [ + 1795102569.987, + "2" + ], + [ + 1795102574.987, + "2" + ], + [ + 1795102579.987, + "2" + ], + [ + 1795102584.987, + "2" + ], + [ + 1795102589.987, + "2" + ] + ] + } + ] + } + }, + "cpu": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "service_name": "messenger-api" + }, + "values": [ + [ + 1795101974.987, + "0.01545454545454548" + ], + [ + 1795101979.987, + "0.10728971154076983" + ], + [ + 1795101984.987, + "0.14900960108606107" + ], + [ + 1795101989.987, + "0.13433264139211398" + ], + [ + 1795101994.987, + "0.12804235887334414" + ], + [ + 1795101999.987, + "0.12438990595696303" + ], + [ + 1795102004.987, + "0.12239784000522154" + ], + [ + 1795102009.987, + "0.1211990419296955" + ], + [ + 1795102014.987, + "0.12122303360580179" + ], + [ + 1795102019.987, + "0.12212899999999999" + ], + [ + 1795102024.987, + "0.1306" + ], + [ + 1795102029.987, + "0.12218181818181817" + ], + [ + 1795102034.987, + "0.08528203076699276" + ], + [ + 1795102039.987, + "0.04454950450040912" + ], + [ + 1795102044.987, + "0.04454707443907052" + ], + [ + 1795102049.987, + "0.0436308106241024" + ], + [ + 1795102054.987, + "0.04436928336333715" + ], + [ + 1795102059.987, + "0.0436339836008945" + ], + [ + 1795102064.987, + "0.04436686304458508" + ], + [ + 1795102069.987, + "0.043454545454545454" + ], + [ + 1795102074.987, + "0.04381818181818182" + ], + [ + 1795102079.987, + "0.04345454545454545" + ], + [ + 1795102084.987, + "0.04454545454545456" + ], + [ + 1795102089.987, + "0.04363636363636364" + ], + [ + 1795102094.987, + "0.04345454545454546" + ], + [ + 1795102099.987, + "0.0429090909090909" + ], + [ + 1795102104.987, + "0.043641918062298814" + ], + [ + 1795102109.987, + "0.04344901557983531" + ], + [ + 1795102114.987, + "0.04472564634013308" + ], + [ + 1795102119.987, + "0.04435718440954042" + ], + [ + 1795102124.987, + "0.045275196828917945" + ], + [ + 1795102129.987, + "0.045090909090909084" + ], + [ + 1795102134.987, + "0.046" + ], + [ + 1795102139.987, + "0.045636363636363614" + ], + [ + 1795102144.987, + "0.046727272727272715" + ], + [ + 1795102149.987, + "0.04617510180337404" + ], + [ + 1795102154.987, + "0.047636363636363636" + ], + [ + 1795102159.987, + "0.04781122745782433" + ], + [ + 1795102164.987, + "0.048551633844307454" + ], + [ + 1795102169.987, + "0.048371551708461406" + ], + [ + 1795102174.987, + "0.04890731246136505" + ], + [ + 1795102179.987, + "0.049082877347343144" + ], + [ + 1795102184.987, + "0.049815464611021223" + ], + [ + 1795102189.987, + "0.04981274770025087" + ], + [ + 1795102194.987, + "0.05163636363636365" + ], + [ + 1795102199.987, + "0.052719604421175126" + ], + [ + 1795102204.987, + "0.054361659576015436" + ], + [ + 1795102209.987, + "0.05435573007562535" + ], + [ + 1795102214.987, + "0.056181818181818194" + ], + [ + 1795102219.987, + "0.056177732528543356" + ], + [ + 1795102224.987, + "0.05726960347617402" + ], + [ + 1795102229.987, + "0.05763636363636361" + ], + [ + 1795102234.987, + "0.05800632796305051" + ], + [ + 1795102239.987, + "0.05781713059762551" + ], + [ + 1795102244.987, + "0.05855184201912936" + ], + [ + 1795102249.987, + "0.05690909090909092" + ], + [ + 1795102254.987, + "0.056371835903404124" + ], + [ + 1795102259.987, + "0.05564041021165173" + ], + [ + 1795102264.987, + "0.054553389583939446" + ], + [ + 1795102269.987, + "0.05218656241476497" + ], + [ + 1795102274.987, + "0.051094625063641" + ], + [ + 1795102279.987, + "0.04927541502263762" + ], + [ + 1795102284.987, + "0.048190580105473724" + ], + [ + 1795102289.987, + "0.04636616542720516" + ], + [ + 1795102294.987, + "0.04545785148010764" + ], + [ + 1795102299.987, + "0.04327272727272728" + ], + [ + 1795102304.987, + "0.04254545454545451" + ], + [ + 1795102309.987, + "0.040727272727272695" + ], + [ + 1795102314.987, + "0.040186202131141595" + ], + [ + 1795102319.987, + "0.039636363636363664" + ], + [ + 1795102324.987, + "0.039093041438623896" + ], + [ + 1795102329.987, + "0.03909090909090909" + ], + [ + 1795102334.987, + "0.04036363636363637" + ], + [ + 1795102339.987, + "0.04090909090909091" + ], + [ + 1795102344.987, + "0.041999999999999975" + ], + [ + 1795102349.987, + "0.041999999999999975" + ], + [ + 1795102354.987, + "0.04418181818181814" + ], + [ + 1795102359.987, + "0.044181818181818176" + ], + [ + 1795102364.987, + "0.04672727272727276" + ], + [ + 1795102369.987, + "0.048179190225987684" + ], + [ + 1795102374.987, + "0.04927272727272729" + ], + [ + 1795102379.987, + "0.051090909090909103" + ], + [ + 1795102384.987, + "0.05254545454545455" + ], + [ + 1795102389.987, + "0.05236078032107335" + ], + [ + 1795102394.987, + "0.05345454545454544" + ], + [ + 1795102399.987, + "0.05363636363636362" + ], + [ + 1795102404.987, + "0.054363636363636364" + ], + [ + 1795102409.987, + "0.05327272727272733" + ], + [ + 1795102414.987, + "0.05454545454545454" + ], + [ + 1795102419.987, + "0.053272727272727236" + ], + [ + 1795102424.987, + "0.05218466461807006" + ], + [ + 1795102429.987, + "0.052364588447062696" + ], + [ + 1795102434.987, + "0.0521818181818182" + ], + [ + 1795102439.987, + "0.0510909090909091" + ], + [ + 1795102444.987, + "0.05200378209324317" + ], + [ + 1795102449.987, + "0.05145454545454545" + ], + [ + 1795102454.987, + "0.051454545454545454" + ], + [ + 1795102459.987, + "0.05145454545454549" + ], + [ + 1795102464.987, + "0.05218181818181816" + ], + [ + 1795102469.987, + "0.051633547261058524" + ], + [ + 1795102474.987, + "0.052181818181818204" + ], + [ + 1795102479.987, + "0.0532727272727273" + ], + [ + 1795102484.987, + "0.051271795058271674" + ], + [ + 1795102489.987, + "0.047818181818181794" + ], + [ + 1795102494.987, + "0.04563719340351643" + ], + [ + 1795102499.987, + "0.04327194051017256" + ], + [ + 1795102504.987, + "0.04218181818181818" + ], + [ + 1795102509.987, + "0.04127347769959454" + ], + [ + 1795102514.987, + "0.040181818181818124" + ], + [ + 1795102519.987, + "0.03909090909090909" + ], + [ + 1795102524.987, + "0.03818459524329038" + ], + [ + 1795102529.987, + "0.0358181818181818" + ], + [ + 1795102534.987, + "0.034000000000000016" + ], + [ + 1795102539.987, + "0.03472790414371171" + ], + [ + 1795102544.987, + "0.03727272727272732" + ], + [ + 1795102549.987, + "0.0396298787471141" + ], + [ + 1795102554.987, + "0.04054545454545448" + ], + [ + 1795102559.987, + "0.04054324309583115" + ], + [ + 1795102564.987, + "0.04145379174924098" + ], + [ + 1795102569.987, + "0.041996182165257745" + ], + [ + 1795102574.987, + "0.042906750540879585" + ], + [ + 1795102579.987, + "0.043813402174308326" + ], + [ + 1795102584.987, + "0.04654122352513409" + ], + [ + 1795102589.987, + "0.047455408280150574" + ] + ] + }, + { + "metric": { + "service_name": "messenger-publisher" + }, + "values": [ + [ + 1795101974.987, + "0.006000000000000001" + ], + [ + 1795101979.987, + "0.006916141971330594" + ], + [ + 1795101984.987, + "0.006278647583048225" + ], + [ + 1795101989.987, + "0.006708396422207033" + ], + [ + 1795101994.987, + "0.006967892430495153" + ], + [ + 1795101999.987, + "0.006916787406048973" + ], + [ + 1795102004.987, + "0.006865828551935241" + ], + [ + 1795102009.987, + "0.007003355159394407" + ], + [ + 1795102014.987, + "0.007100645576581757" + ], + [ + 1795102019.987, + "0.007948077777777778" + ], + [ + 1795102024.987, + "0.009199999999999998" + ], + [ + 1795102029.987, + "0.009272727272727274" + ], + [ + 1795102034.987, + "0.009455576972033313" + ], + [ + 1795102039.987, + "0.010000909173561235" + ], + [ + 1795102044.987, + "0.009818538855958396" + ], + [ + 1795102049.987, + "0.009635137346155943" + ], + [ + 1795102054.987, + "0.009819431564017237" + ], + [ + 1795102059.987, + "0.010181262840208717" + ], + [ + 1795102064.987, + "0.009818895919703252" + ], + [ + 1795102069.987, + "0.009636363636363636" + ], + [ + 1795102074.987, + "0.009272727272727273" + ], + [ + 1795102079.987, + "0.009454545454545455" + ], + [ + 1795102084.987, + "0.009090909090909087" + ], + [ + 1795102089.987, + "0.00909090909090909" + ], + [ + 1795102094.987, + "0.008909090909090908" + ], + [ + 1795102099.987, + "0.008545454545454549" + ], + [ + 1795102104.987, + "0.008364700961940613" + ], + [ + 1795102109.987, + "0.008362572036286296" + ], + [ + 1795102114.987, + "0.007817897530998876" + ], + [ + 1795102119.987, + "0.008180628272251309" + ], + [ + 1795102124.987, + "0.008182264487153845" + ], + [ + 1795102129.987, + "0.008545454545454545" + ], + [ + 1795102134.987, + "0.008181818181818184" + ], + [ + 1795102139.987, + "0.008545454545454545" + ], + [ + 1795102144.987, + "0.008363636363636363" + ], + [ + 1795102149.987, + "0.008362420011634671" + ], + [ + 1795102154.987, + "0.008727272727272722" + ], + [ + 1795102159.987, + "0.008907795229784759" + ], + [ + 1795102164.987, + "0.008910224937719341" + ], + [ + 1795102169.987, + "0.009092396937680708" + ], + [ + 1795102174.987, + "0.009090578524417291" + ], + [ + 1795102179.987, + "0.009089421730989478" + ], + [ + 1795102184.987, + "0.008908604985182624" + ], + [ + 1795102189.987, + "0.008908119114278447" + ], + [ + 1795102194.987, + "0.008909090909090913" + ], + [ + 1795102199.987, + "0.008907795229784762" + ], + [ + 1795102204.987, + "0.008908766953928952" + ], + [ + 1795102209.987, + "0.009089586969168121" + ], + [ + 1795102214.987, + "0.008909090909090913" + ], + [ + 1795102219.987, + "0.008908443022325652" + ], + [ + 1795102224.987, + "0.008908604985182632" + ], + [ + 1795102229.987, + "0.00909090909090909" + ], + [ + 1795102234.987, + "0.009273738953340362" + ], + [ + 1795102239.987, + "0.00945437355684443" + ], + [ + 1795102244.987, + "0.009455576972033313" + ], + [ + 1795102249.987, + "0.009636363636363632" + ], + [ + 1795102254.987, + "0.01000145475705557" + ], + [ + 1795102259.987, + "0.010182558731544113" + ], + [ + 1795102264.987, + "0.010183299389002039" + ], + [ + 1795102269.987, + "0.010546413310300932" + ], + [ + 1795102274.987, + "0.010546221543384967" + ], + [ + 1795102279.987, + "0.010909685982871795" + ], + [ + 1795102284.987, + "0.010729223495180946" + ], + [ + 1795102289.987, + "0.010727857883157268" + ], + [ + 1795102294.987, + "0.010728052949305393" + ], + [ + 1795102299.987, + "0.010545454545454538" + ], + [ + 1795102304.987, + "0.010545454545454547" + ], + [ + 1795102309.987, + "0.010545454545454547" + ], + [ + 1795102314.987, + "0.010364767065498052" + ], + [ + 1795102319.987, + "0.010363636363636368" + ], + [ + 1795102324.987, + "0.010182373584013674" + ], + [ + 1795102329.987, + "0.010000000000000004" + ], + [ + 1795102334.987, + "0.009818181818181811" + ], + [ + 1795102339.987, + "0.009818181818181811" + ], + [ + 1795102344.987, + "0.009999999999999997" + ], + [ + 1795102349.987, + "0.010000000000000004" + ], + [ + 1795102354.987, + "0.010181818181818183" + ], + [ + 1795102359.987, + "0.009999999999999997" + ], + [ + 1795102364.987, + "0.010181818181818183" + ], + [ + 1795102369.987, + "0.010363071105212441" + ], + [ + 1795102374.987, + "0.010181818181818174" + ], + [ + 1795102379.987, + "0.010545454545454547" + ], + [ + 1795102384.987, + "0.011090909090909096" + ], + [ + 1795102389.987, + "0.011090304165227358" + ], + [ + 1795102394.987, + "0.011090909090909096" + ], + [ + 1795102399.987, + "0.011090909090909096" + ], + [ + 1795102404.987, + "0.011090909090909096" + ], + [ + 1795102409.987, + "0.011454545454545452" + ], + [ + 1795102414.987, + "0.011636363636363646" + ], + [ + 1795102419.987, + "0.01163636363636363" + ], + [ + 1795102424.987, + "0.011636998381729923" + ], + [ + 1795102429.987, + "0.011818396698121791" + ], + [ + 1795102434.987, + "0.011454545454545452" + ], + [ + 1795102439.987, + "0.011272727272727257" + ], + [ + 1795102444.987, + "0.011455378572987123" + ], + [ + 1795102449.987, + "0.011818181818181825" + ], + [ + 1795102454.987, + "0.01163636363636363" + ], + [ + 1795102459.987, + "0.01163636363636363" + ], + [ + 1795102464.987, + "0.011454545454545452" + ], + [ + 1795102469.987, + "0.01145392069523479" + ], + [ + 1795102474.987, + "0.011454545454545452" + ], + [ + 1795102479.987, + "0.011272727272727257" + ], + [ + 1795102484.987, + "0.01090889256558971" + ], + [ + 1795102489.987, + "0.010363636363636353" + ], + [ + 1795102494.987, + "0.00963653884616086" + ], + [ + 1795102499.987, + "0.009272558680751256" + ], + [ + 1795102504.987, + "0.008909090909090896" + ], + [ + 1795102509.987, + "0.008727431407843772" + ], + [ + 1795102514.987, + "0.008909090909090913" + ], + [ + 1795102519.987, + "0.00854545454545454" + ], + [ + 1795102524.987, + "0.008546076078260249" + ], + [ + 1795102529.987, + "0.008181818181818184" + ], + [ + 1795102534.987, + "0.008363636363636363" + ], + [ + 1795102539.987, + "0.008363788432516955" + ], + [ + 1795102544.987, + "0.009090909090909106" + ], + [ + 1795102549.987, + "0.009452998600229046" + ], + [ + 1795102554.987, + "0.009818181818181818" + ], + [ + 1795102559.987, + "0.009999454575204987" + ], + [ + 1795102564.987, + "0.010181633061217077" + ], + [ + 1795102569.987, + "0.009817289337332955" + ], + [ + 1795102574.987, + "0.01036307110521245" + ], + [ + 1795102579.987, + "0.00999890920990437" + ], + [ + 1795102584.987, + "0.010180892646123072" + ], + [ + 1795102589.987, + "0.01018200330915107" + ] + ] + }, + { + "metric": { + "service_name": "messenger-worker" + }, + "values": [ + [ + 1795101974.987, + "0.003454545454545453" + ], + [ + 1795101979.987, + "0.004734522118589949" + ], + [ + 1795101984.987, + "0.005327211038584559" + ], + [ + 1795101989.987, + "0.005408429774431247" + ], + [ + 1795101994.987, + "0.005649541774668826" + ], + [ + 1795101999.987, + "0.006139046632203438" + ], + [ + 1795102004.987, + "0.006480387410937802" + ], + [ + 1795102009.987, + "0.0065829574547744" + ], + [ + 1795102014.987, + "0.007082942985292153" + ], + [ + 1795102019.987, + "0.007172655555555555" + ], + [ + 1795102024.987, + "0.008399999999999998" + ], + [ + 1795102029.987, + "0.008363636363636363" + ], + [ + 1795102034.987, + "0.008364548859875622" + ], + [ + 1795102039.987, + "0.008000727338848984" + ], + [ + 1795102044.987, + "0.008182115713298665" + ], + [ + 1795102049.987, + "0.008180776992019195" + ], + [ + 1795102054.987, + "0.008364700961940606" + ], + [ + 1795102059.987, + "0.007999563660163989" + ], + [ + 1795102064.987, + "0.008546076078260235" + ], + [ + 1795102069.987, + "0.00818181818181818" + ], + [ + 1795102074.987, + "0.008363636363636363" + ], + [ + 1795102079.987, + "0.008363636363636365" + ], + [ + 1795102084.987, + "0.008727272727272722" + ], + [ + 1795102089.987, + "0.008909090909090906" + ], + [ + 1795102094.987, + "0.009272727272727274" + ], + [ + 1795102099.987, + "0.009454545454545452" + ], + [ + 1795102104.987, + "0.009273907588238504" + ], + [ + 1795102109.987, + "0.009271547257621764" + ], + [ + 1795102114.987, + "0.009817824806370681" + ], + [ + 1795102119.987, + "0.00963496218731821" + ], + [ + 1795102124.987, + "0.010182373584013669" + ], + [ + 1795102129.987, + "0.010363636363636363" + ], + [ + 1795102134.987, + "0.01018181818181818" + ], + [ + 1795102139.987, + "0.01" + ], + [ + 1795102144.987, + "0.010181818181818186" + ], + [ + 1795102149.987, + "0.00999854566608493" + ], + [ + 1795102154.987, + "0.009999999999999997" + ], + [ + 1795102159.987, + "0.01054392088423502" + ], + [ + 1795102164.987, + "0.01036495553979597" + ], + [ + 1795102169.987, + "0.010365332508956009" + ], + [ + 1795102174.987, + "0.010181447947347363" + ], + [ + 1795102179.987, + "0.009998363904088422" + ], + [ + 1795102184.987, + "0.009999454575204986" + ], + [ + 1795102189.987, + "0.009635312511362393" + ], + [ + 1795102194.987, + "0.00981818181818182" + ], + [ + 1795102199.987, + "0.009634962187318205" + ], + [ + 1795102204.987, + "0.009454201665393987" + ], + [ + 1795102209.987, + "0.00945317044793485" + ], + [ + 1795102214.987, + "0.00909090909090909" + ], + [ + 1795102219.987, + "0.009272052941604247" + ], + [ + 1795102224.987, + "0.008908604985182627" + ], + [ + 1795102229.987, + "0.009090909090909094" + ], + [ + 1795102234.987, + "0.009273738953340369" + ], + [ + 1795102239.987, + "0.00927255868075126" + ], + [ + 1795102244.987, + "0.010182929046805109" + ], + [ + 1795102249.987, + "0.009636363636363637" + ], + [ + 1795102254.987, + "0.009637765493162642" + ], + [ + 1795102259.987, + "0.009818895919703252" + ], + [ + 1795102264.987, + "0.010001454757055573" + ], + [ + 1795102269.987, + "0.010000909173561235" + ], + [ + 1795102274.987, + "0.01036439013746454" + ], + [ + 1795102279.987, + "0.010000545484299145" + ], + [ + 1795102284.987, + "0.01018366975813784" + ], + [ + 1795102289.987, + "0.009818717384584615" + ], + [ + 1795102294.987, + "0.009818895919703252" + ], + [ + 1795102299.987, + "0.009272727272727273" + ], + [ + 1795102304.987, + "0.009818181818181823" + ], + [ + 1795102309.987, + "0.009454545454545459" + ], + [ + 1795102314.987, + "0.009273738953340365" + ], + [ + 1795102319.987, + "0.009090909090909094" + ], + [ + 1795102324.987, + "0.009091404985726498" + ], + [ + 1795102329.987, + "0.00872727272727273" + ], + [ + 1795102334.987, + "0.009454545454545459" + ], + [ + 1795102339.987, + "0.008909090909090922" + ], + [ + 1795102344.987, + "0.009090909090909094" + ], + [ + 1795102349.987, + "0.009090909090909087" + ], + [ + 1795102354.987, + "0.009090909090909087" + ], + [ + 1795102359.987, + "0.008545454545454549" + ], + [ + 1795102364.987, + "0.008000000000000004" + ], + [ + 1795102369.987, + "0.007454138865152817" + ], + [ + 1795102374.987, + "0.006363636363636365" + ], + [ + 1795102379.987, + "0.005636363636363637" + ], + [ + 1795102384.987, + "0.004727272727272731" + ], + [ + 1795102389.987, + "0.003510600000000003" + ], + [ + 1795102394.987, + "0.0030007518518518506" + ], + [ + 1795102399.987, + "0.002132950000000002" + ], + [ + 1795102404.987, + "0.001255300000000013" + ], + [ + 1795102409.987, + "0.0005443166666666793" + ], + [ + 1795102414.987, + "0" + ], + [ + 1795102419.987, + "0" + ], + [ + 1795102424.987, + "0" + ], + [ + 1795102429.987, + "0" + ], + [ + 1795102434.987, + "0" + ], + [ + 1795102494.987, + "0.006994348869773947" + ], + [ + 1795102499.987, + "0.007514433333333328" + ], + [ + 1795102504.987, + "0.007695233333333329" + ], + [ + 1795102509.987, + "0.007746968598429918" + ], + [ + 1795102514.987, + "0.008624266666666665" + ], + [ + 1795102519.987, + "0.00933525" + ], + [ + 1795102524.987, + "0.010081461511145213" + ], + [ + 1795102529.987, + "0.010848979166666665" + ], + [ + 1795102534.987, + "0.011437477777777774" + ], + [ + 1795102539.987, + "0.012600252005040098" + ], + [ + 1795102544.987, + "0.01236363636363636" + ], + [ + 1795102549.987, + "0.010361940773328002" + ], + [ + 1795102554.987, + "0.009636363636363639" + ], + [ + 1795102559.987, + "0.009454029780193812" + ], + [ + 1795102564.987, + "0.009818003309030744" + ], + [ + 1795102569.987, + "0.00945368602854286" + ], + [ + 1795102574.987, + "0.009817646310201263" + ], + [ + 1795102579.987, + "0.009817110860633388" + ], + [ + 1795102584.987, + "0.009817289337332972" + ], + [ + 1795102589.987, + "0.01036382479681449" + ] + ] + } + ] + } + }, + "rss": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "service_name": "messenger-api" + }, + "values": [ + [ + 1795101974.987, + "430960640" + ], + [ + 1795101979.987, + "492179456" + ], + [ + 1795101984.987, + "632377344" + ], + [ + 1795101989.987, + "633081856" + ], + [ + 1795101994.987, + "383545344" + ], + [ + 1795101999.987, + "384172032" + ], + [ + 1795102004.987, + "384483328" + ], + [ + 1795102009.987, + "384499712" + ], + [ + 1795102014.987, + "383135744" + ], + [ + 1795102019.987, + "383365120" + ], + [ + 1795102024.987, + "383385600" + ], + [ + 1795102029.987, + "383262720" + ], + [ + 1795102034.987, + "383234048" + ], + [ + 1795102039.987, + "384290816" + ], + [ + 1795102044.987, + "383795200" + ], + [ + 1795102049.987, + "384327680" + ], + [ + 1795102054.987, + "383676416" + ], + [ + 1795102059.987, + "383983616" + ], + [ + 1795102064.987, + "384532480" + ], + [ + 1795102069.987, + "384294912" + ], + [ + 1795102074.987, + "384290816" + ], + [ + 1795102079.987, + "384299008" + ], + [ + 1795102084.987, + "384221184" + ], + [ + 1795102089.987, + "384659456" + ], + [ + 1795102094.987, + "384774144" + ], + [ + 1795102099.987, + "384892928" + ], + [ + 1795102104.987, + "384819200" + ], + [ + 1795102109.987, + "385757184" + ], + [ + 1795102114.987, + "385179648" + ], + [ + 1795102119.987, + "385347584" + ], + [ + 1795102124.987, + "384872448" + ], + [ + 1795102129.987, + "384466944" + ], + [ + 1795102134.987, + "384712704" + ], + [ + 1795102139.987, + "384716800" + ], + [ + 1795102144.987, + "384585728" + ], + [ + 1795102149.987, + "384610304" + ], + [ + 1795102154.987, + "384757760" + ], + [ + 1795102159.987, + "384761856" + ], + [ + 1795102164.987, + "384544768" + ], + [ + 1795102169.987, + "384905216" + ], + [ + 1795102174.987, + "385957888" + ], + [ + 1795102179.987, + "386408448" + ], + [ + 1795102184.987, + "385953792" + ], + [ + 1795102189.987, + "386781184" + ], + [ + 1795102194.987, + "387055616" + ], + [ + 1795102199.987, + "387104768" + ], + [ + 1795102204.987, + "386433024" + ], + [ + 1795102209.987, + "386560000" + ], + [ + 1795102214.987, + "387264512" + ], + [ + 1795102219.987, + "386355200" + ], + [ + 1795102224.987, + "387170304" + ], + [ + 1795102229.987, + "387502080" + ], + [ + 1795102234.987, + "387735552" + ], + [ + 1795102239.987, + "387780608" + ], + [ + 1795102244.987, + "386867200" + ], + [ + 1795102249.987, + "386912256" + ], + [ + 1795102254.987, + "387104768" + ], + [ + 1795102259.987, + "386916352" + ], + [ + 1795102264.987, + "386859008" + ], + [ + 1795102269.987, + "386834432" + ], + [ + 1795102274.987, + "387530752" + ], + [ + 1795102279.987, + "387440640" + ], + [ + 1795102284.987, + "387444736" + ], + [ + 1795102289.987, + "387497984" + ], + [ + 1795102294.987, + "387592192" + ], + [ + 1795102299.987, + "386379776" + ], + [ + 1795102304.987, + "387379200" + ], + [ + 1795102309.987, + "387403776" + ], + [ + 1795102314.987, + "386539520" + ], + [ + 1795102319.987, + "387145728" + ], + [ + 1795102324.987, + "386969600" + ], + [ + 1795102329.987, + "386973696" + ], + [ + 1795102334.987, + "386920448" + ], + [ + 1795102339.987, + "386551808" + ], + [ + 1795102344.987, + "386785280" + ], + [ + 1795102349.987, + "386953216" + ], + [ + 1795102354.987, + "386703360" + ], + [ + 1795102359.987, + "387108864" + ], + [ + 1795102364.987, + "387100672" + ], + [ + 1795102369.987, + "387268608" + ], + [ + 1795102374.987, + "386019328" + ], + [ + 1795102379.987, + "386891776" + ], + [ + 1795102384.987, + "386895872" + ], + [ + 1795102389.987, + "386895872" + ], + [ + 1795102394.987, + "386269184" + ], + [ + 1795102399.987, + "387035136" + ], + [ + 1795102404.987, + "387182592" + ], + [ + 1795102409.987, + "387190784" + ], + [ + 1795102414.987, + "387125248" + ], + [ + 1795102419.987, + "387174400" + ], + [ + 1795102424.987, + "387395584" + ], + [ + 1795102429.987, + "386932736" + ], + [ + 1795102434.987, + "386555904" + ], + [ + 1795102439.987, + "386633728" + ], + [ + 1795102444.987, + "387301376" + ], + [ + 1795102449.987, + "385859584" + ], + [ + 1795102454.987, + "387063808" + ], + [ + 1795102459.987, + "387072000" + ], + [ + 1795102464.987, + "387272704" + ], + [ + 1795102469.987, + "386859008" + ], + [ + 1795102474.987, + "386850816" + ], + [ + 1795102479.987, + "386887680" + ], + [ + 1795102484.987, + "385933312" + ], + [ + 1795102489.987, + "386752512" + ], + [ + 1795102494.987, + "387239936" + ], + [ + 1795102499.987, + "387457024" + ], + [ + 1795102504.987, + "387104768" + ], + [ + 1795102509.987, + "387276800" + ], + [ + 1795102514.987, + "387338240" + ], + [ + 1795102519.987, + "387338240" + ], + [ + 1795102524.987, + "386449408" + ], + [ + 1795102529.987, + "386936832" + ], + [ + 1795102534.987, + "387461120" + ], + [ + 1795102539.987, + "387502080" + ], + [ + 1795102544.987, + "386523136" + ], + [ + 1795102549.987, + "387366912" + ], + [ + 1795102554.987, + "387420160" + ], + [ + 1795102559.987, + "387432448" + ], + [ + 1795102564.987, + "387371008" + ], + [ + 1795102569.987, + "387379200" + ], + [ + 1795102574.987, + "387457024" + ], + [ + 1795102579.987, + "387137536" + ], + [ + 1795102584.987, + "387112960" + ], + [ + 1795102589.987, + "387133440" + ] + ] + }, + { + "metric": { + "service_name": "messenger-publisher" + }, + "values": [ + [ + 1795101974.987, + "153427968" + ], + [ + 1795101979.987, + "154124288" + ], + [ + 1795101984.987, + "154193920" + ], + [ + 1795101989.987, + "79118336" + ], + [ + 1795101994.987, + "79122432" + ], + [ + 1795101999.987, + "79122432" + ], + [ + 1795102004.987, + "79130624" + ], + [ + 1795102009.987, + "79130624" + ], + [ + 1795102014.987, + "79130624" + ], + [ + 1795102019.987, + "79138816" + ], + [ + 1795102024.987, + "79138816" + ], + [ + 1795102029.987, + "79142912" + ], + [ + 1795102034.987, + "79142912" + ], + [ + 1795102039.987, + "79147008" + ], + [ + 1795102044.987, + "79151104" + ], + [ + 1795102049.987, + "79155200" + ], + [ + 1795102054.987, + "74833920" + ], + [ + 1795102059.987, + "74833920" + ], + [ + 1795102064.987, + "74846208" + ], + [ + 1795102069.987, + "74854400" + ], + [ + 1795102074.987, + "74854400" + ], + [ + 1795102079.987, + "74858496" + ], + [ + 1795102084.987, + "74858496" + ], + [ + 1795102089.987, + "74862592" + ], + [ + 1795102094.987, + "74866688" + ], + [ + 1795102099.987, + "74866688" + ], + [ + 1795102104.987, + "74866688" + ], + [ + 1795102109.987, + "74866688" + ], + [ + 1795102114.987, + "74870784" + ], + [ + 1795102119.987, + "74874880" + ], + [ + 1795102124.987, + "74883072" + ], + [ + 1795102129.987, + "74887168" + ], + [ + 1795102134.987, + "74895360" + ], + [ + 1795102139.987, + "74895360" + ], + [ + 1795102144.987, + "74899456" + ], + [ + 1795102149.987, + "74903552" + ], + [ + 1795102154.987, + "74911744" + ], + [ + 1795102159.987, + "74911744" + ], + [ + 1795102164.987, + "74915840" + ], + [ + 1795102169.987, + "74915840" + ], + [ + 1795102174.987, + "74915840" + ], + [ + 1795102179.987, + "74924032" + ], + [ + 1795102184.987, + "74924032" + ], + [ + 1795102189.987, + "74924032" + ], + [ + 1795102194.987, + "74924032" + ], + [ + 1795102199.987, + "74928128" + ], + [ + 1795102204.987, + "74919936" + ], + [ + 1795102209.987, + "74936320" + ], + [ + 1795102214.987, + "74936320" + ], + [ + 1795102219.987, + "74940416" + ], + [ + 1795102224.987, + "74940416" + ], + [ + 1795102229.987, + "74944512" + ], + [ + 1795102234.987, + "74948608" + ], + [ + 1795102239.987, + "74952704" + ], + [ + 1795102244.987, + "74952704" + ], + [ + 1795102249.987, + "74956800" + ], + [ + 1795102254.987, + "74956800" + ], + [ + 1795102259.987, + "74964992" + ], + [ + 1795102264.987, + "74964992" + ], + [ + 1795102269.987, + "74964992" + ], + [ + 1795102274.987, + "74969088" + ], + [ + 1795102279.987, + "74960896" + ], + [ + 1795102284.987, + "74964992" + ], + [ + 1795102289.987, + "74977280" + ], + [ + 1795102294.987, + "74981376" + ], + [ + 1795102299.987, + "74981376" + ], + [ + 1795102304.987, + "74985472" + ], + [ + 1795102309.987, + "74989568" + ], + [ + 1795102314.987, + "74989568" + ], + [ + 1795102319.987, + "74993664" + ], + [ + 1795102324.987, + "74997760" + ], + [ + 1795102329.987, + "74997760" + ], + [ + 1795102334.987, + "75001856" + ], + [ + 1795102339.987, + "75001856" + ], + [ + 1795102344.987, + "75005952" + ], + [ + 1795102349.987, + "75010048" + ], + [ + 1795102354.987, + "75001856" + ], + [ + 1795102359.987, + "75005952" + ], + [ + 1795102364.987, + "75005952" + ], + [ + 1795102369.987, + "75022336" + ], + [ + 1795102374.987, + "75022336" + ], + [ + 1795102379.987, + "75026432" + ], + [ + 1795102384.987, + "75026432" + ], + [ + 1795102389.987, + "75030528" + ], + [ + 1795102394.987, + "75030528" + ], + [ + 1795102399.987, + "75034624" + ], + [ + 1795102404.987, + "75034624" + ], + [ + 1795102409.987, + "75034624" + ], + [ + 1795102414.987, + "75034624" + ], + [ + 1795102419.987, + "75042816" + ], + [ + 1795102424.987, + "75042816" + ], + [ + 1795102429.987, + "75034624" + ], + [ + 1795102434.987, + "75034624" + ], + [ + 1795102439.987, + "75034624" + ], + [ + 1795102444.987, + "75034624" + ], + [ + 1795102449.987, + "75046912" + ], + [ + 1795102454.987, + "75046912" + ], + [ + 1795102459.987, + "75046912" + ], + [ + 1795102464.987, + "75046912" + ], + [ + 1795102469.987, + "75046912" + ], + [ + 1795102474.987, + "75046912" + ], + [ + 1795102479.987, + "75051008" + ], + [ + 1795102484.987, + "75051008" + ], + [ + 1795102489.987, + "75051008" + ], + [ + 1795102494.987, + "75051008" + ], + [ + 1795102499.987, + "75051008" + ], + [ + 1795102504.987, + "75042816" + ], + [ + 1795102509.987, + "75046912" + ], + [ + 1795102514.987, + "75046912" + ], + [ + 1795102519.987, + "75046912" + ], + [ + 1795102524.987, + "75046912" + ], + [ + 1795102529.987, + "75059200" + ], + [ + 1795102534.987, + "75059200" + ], + [ + 1795102539.987, + "75059200" + ], + [ + 1795102544.987, + "75059200" + ], + [ + 1795102549.987, + "75059200" + ], + [ + 1795102554.987, + "75059200" + ], + [ + 1795102559.987, + "75063296" + ], + [ + 1795102564.987, + "75063296" + ], + [ + 1795102569.987, + "75063296" + ], + [ + 1795102574.987, + "75063296" + ], + [ + 1795102579.987, + "75055104" + ], + [ + 1795102584.987, + "75055104" + ], + [ + 1795102589.987, + "75059200" + ] + ] + }, + { + "metric": { + "service_name": "messenger-worker" + }, + "values": [ + [ + 1795101974.987, + "332845056" + ], + [ + 1795101979.987, + "334532608" + ], + [ + 1795101984.987, + "334913536" + ], + [ + 1795101989.987, + "334913536" + ], + [ + 1795101994.987, + "168878080" + ], + [ + 1795101999.987, + "168886272" + ], + [ + 1795102004.987, + "168898560" + ], + [ + 1795102009.987, + "168923136" + ], + [ + 1795102014.987, + "168927232" + ], + [ + 1795102019.987, + "168931328" + ], + [ + 1795102024.987, + "168939520" + ], + [ + 1795102029.987, + "165466112" + ], + [ + 1795102034.987, + "165810176" + ], + [ + 1795102039.987, + "165826560" + ], + [ + 1795102044.987, + "165830656" + ], + [ + 1795102049.987, + "162357248" + ], + [ + 1795102054.987, + "162709504" + ], + [ + 1795102059.987, + "162717696" + ], + [ + 1795102064.987, + "162721792" + ], + [ + 1795102069.987, + "162721792" + ], + [ + 1795102074.987, + "162734080" + ], + [ + 1795102079.987, + "162742272" + ], + [ + 1795102084.987, + "162750464" + ], + [ + 1795102089.987, + "162742272" + ], + [ + 1795102094.987, + "162775040" + ], + [ + 1795102099.987, + "162799616" + ], + [ + 1795102104.987, + "162717696" + ], + [ + 1795102109.987, + "162807808" + ], + [ + 1795102114.987, + "162824192" + ], + [ + 1795102119.987, + "162836480" + ], + [ + 1795102124.987, + "162852864" + ], + [ + 1795102129.987, + "162861056" + ], + [ + 1795102134.987, + "162885632" + ], + [ + 1795102139.987, + "162906112" + ], + [ + 1795102144.987, + "162918400" + ], + [ + 1795102149.987, + "162926592" + ], + [ + 1795102154.987, + "162816000" + ], + [ + 1795102159.987, + "162897920" + ], + [ + 1795102164.987, + "162902016" + ], + [ + 1795102169.987, + "162910208" + ], + [ + 1795102174.987, + "162865152" + ], + [ + 1795102179.987, + "162914304" + ], + [ + 1795102184.987, + "162926592" + ], + [ + 1795102189.987, + "162930688" + ], + [ + 1795102194.987, + "162934784" + ], + [ + 1795102199.987, + "162938880" + ], + [ + 1795102204.987, + "162861056" + ], + [ + 1795102209.987, + "162934784" + ], + [ + 1795102214.987, + "162959360" + ], + [ + 1795102219.987, + "162963456" + ], + [ + 1795102224.987, + "162971648" + ], + [ + 1795102229.987, + "162861056" + ], + [ + 1795102234.987, + "162942976" + ], + [ + 1795102239.987, + "162947072" + ], + [ + 1795102244.987, + "162951168" + ], + [ + 1795102249.987, + "162951168" + ], + [ + 1795102254.987, + "162959360" + ], + [ + 1795102259.987, + "162959360" + ], + [ + 1795102264.987, + "162664448" + ], + [ + 1795102269.987, + "162672640" + ], + [ + 1795102274.987, + "162955264" + ], + [ + 1795102279.987, + "162975744" + ], + [ + 1795102284.987, + "162983936" + ], + [ + 1795102289.987, + "162668544" + ], + [ + 1795102294.987, + "162676736" + ], + [ + 1795102299.987, + "163004416" + ], + [ + 1795102304.987, + "163020800" + ], + [ + 1795102309.987, + "163020800" + ], + [ + 1795102314.987, + "163028992" + ], + [ + 1795102319.987, + "163028992" + ], + [ + 1795102324.987, + "163037184" + ], + [ + 1795102329.987, + "162951168" + ], + [ + 1795102334.987, + "163033088" + ], + [ + 1795102339.987, + "163037184" + ], + [ + 1795102344.987, + "163041280" + ], + [ + 1795102349.987, + "163049472" + ], + [ + 1795102354.987, + "163061760" + ], + [ + 1795102359.987, + "163065856" + ], + [ + 1795102364.987, + "163065856" + ], + [ + 1795102369.987, + "163065856" + ], + [ + 1795102374.987, + "163065856" + ], + [ + 1795102379.987, + "163065856" + ], + [ + 1795102384.987, + "163065856" + ], + [ + 1795102489.987, + "168800256" + ], + [ + 1795102494.987, + "169136128" + ], + [ + 1795102499.987, + "169254912" + ], + [ + 1795102504.987, + "165920768" + ], + [ + 1795102509.987, + "162516992" + ], + [ + 1795102514.987, + "162521088" + ], + [ + 1795102519.987, + "162529280" + ], + [ + 1795102524.987, + "162549760" + ], + [ + 1795102529.987, + "162852864" + ], + [ + 1795102534.987, + "162877440" + ], + [ + 1795102539.987, + "162885632" + ], + [ + 1795102544.987, + "162897920" + ], + [ + 1795102549.987, + "162910208" + ], + [ + 1795102554.987, + "162918400" + ], + [ + 1795102559.987, + "162836480" + ], + [ + 1795102564.987, + "162914304" + ], + [ + 1795102569.987, + "162922496" + ], + [ + 1795102574.987, + "162930688" + ], + [ + 1795102579.987, + "162889728" + ], + [ + 1795102584.987, + "162910208" + ], + [ + 1795102589.987, + "162918400" + ] + ] + } + ] + } + }, + "api_bad": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "__name__": "lab6:api_bad_ratio:2m" + }, + "values": [ + [ + 1795101974.987, + "0" + ], + [ + 1795101979.987, + "0" + ], + [ + 1795101984.987, + "0" + ], + [ + 1795101989.987, + "0" + ], + [ + 1795101994.987, + "0" + ], + [ + 1795101999.987, + "0" + ], + [ + 1795102004.987, + "0" + ], + [ + 1795102009.987, + "0" + ], + [ + 1795102014.987, + "0" + ], + [ + 1795102019.987, + "0" + ], + [ + 1795102024.987, + "0" + ], + [ + 1795102029.987, + "0" + ], + [ + 1795102034.987, + "0" + ], + [ + 1795102039.987, + "0" + ], + [ + 1795102044.987, + "0" + ], + [ + 1795102049.987, + "0" + ], + [ + 1795102054.987, + "0" + ], + [ + 1795102059.987, + "0" + ], + [ + 1795102064.987, + "0" + ], + [ + 1795102069.987, + "0" + ], + [ + 1795102074.987, + "0" + ], + [ + 1795102079.987, + "0" + ], + [ + 1795102084.987, + "0" + ], + [ + 1795102089.987, + "0" + ], + [ + 1795102094.987, + "0" + ], + [ + 1795102099.987, + "0" + ], + [ + 1795102104.987, + "0" + ], + [ + 1795102109.987, + "0.013892844416217593" + ], + [ + 1795102114.987, + "0.01384380420169672" + ], + [ + 1795102119.987, + "0.013892386273567859" + ], + [ + 1795102124.987, + "0.013844146632578701" + ], + [ + 1795102129.987, + "0.013892500809270198" + ], + [ + 1795102134.987, + "0.01384460320671808" + ], + [ + 1795102139.987, + "0.013892844416217595" + ], + [ + 1795102144.987, + "0.01384460320671808" + ], + [ + 1795102149.987, + "0.013892844416217595" + ], + [ + 1795102154.987, + "0.01384460320671808" + ], + [ + 1795102159.987, + "0.013892844416217595" + ], + [ + 1795102164.987, + "0.013843804201696721" + ], + [ + 1795102169.987, + "0.013990342485717481" + ], + [ + 1795102174.987, + "0.042120294900170975" + ], + [ + 1795102179.987, + "0.07750664301250751" + ], + [ + 1795102184.987, + "0.11235016799594966" + ], + [ + 1795102189.987, + "0.14800105492317514" + ], + [ + 1795102194.987, + "0.18261468409231948" + ], + [ + 1795102199.987, + "0.21852770619847792" + ], + [ + 1795102204.987, + "0.25290872236114675" + ], + [ + 1795102209.987, + "0.2890866189596236" + ], + [ + 1795102214.987, + "0.3232348060339257" + ], + [ + 1795102219.987, + "0.35967781534806287" + ], + [ + 1795102224.987, + "0.3802816901408451" + ], + [ + 1795102229.987, + "0.41696113074204955" + ], + [ + 1795102234.987, + "0.45070422535211263" + ], + [ + 1795102239.987, + "0.4876325088339223" + ], + [ + 1795102244.987, + "0.5211267605633804" + ], + [ + 1795102249.987, + "0.558303886925795" + ], + [ + 1795102254.987, + "0.5915492957746479" + ], + [ + 1795102259.987, + "0.6132404181184669" + ], + [ + 1795102264.987, + "0.6111111111111112" + ], + [ + 1795102269.987, + "0.613240418118467" + ], + [ + 1795102274.987, + "0.6111111111111112" + ], + [ + 1795102279.987, + "0.6132404181184669" + ], + [ + 1795102284.987, + "0.6068965517241379" + ], + [ + 1795102289.987, + "0.577319587628866" + ], + [ + 1795102294.987, + "0.5392491467576792" + ], + [ + 1795102299.987, + "0.5068493150684932" + ], + [ + 1795102304.987, + "0.4709897610921501" + ], + [ + 1795102309.987, + "0.4383561643835617" + ], + [ + 1795102314.987, + "0.40273037542662116" + ], + [ + 1795102319.987, + "0.3698630136986301" + ], + [ + 1795102324.987, + "0.33447098976109213" + ], + [ + 1795102329.987, + "0.30136986301369867" + ], + [ + 1795102334.987, + "0.26621160409556316" + ], + [ + 1795102339.987, + "0.23287671232876714" + ], + [ + 1795102344.987, + "0.1979522184300341" + ], + [ + 1795102349.987, + "0.1643835616438356" + ], + [ + 1795102354.987, + "0.12969283276450513" + ], + [ + 1795102359.987, + "0.0958904109589041" + ], + [ + 1795102364.987, + "0.06143344709897611" + ], + [ + 1795102369.987, + "0.0273972602739726" + ], + [ + 1795102374.987, + "0" + ], + [ + 1795102379.987, + "0" + ], + [ + 1795102384.987, + "0" + ], + [ + 1795102389.987, + "0" + ], + [ + 1795102394.987, + "0" + ], + [ + 1795102399.987, + "0" + ], + [ + 1795102404.987, + "0" + ], + [ + 1795102409.987, + "0" + ], + [ + 1795102414.987, + "0" + ], + [ + 1795102419.987, + "0" + ], + [ + 1795102424.987, + "0" + ], + [ + 1795102429.987, + "0" + ], + [ + 1795102434.987, + "0" + ], + [ + 1795102439.987, + "0" + ], + [ + 1795102444.987, + "0" + ], + [ + 1795102449.987, + "0" + ], + [ + 1795102454.987, + "0" + ], + [ + 1795102459.987, + "0" + ], + [ + 1795102464.987, + "0" + ], + [ + 1795102469.987, + "0" + ], + [ + 1795102474.987, + "0" + ], + [ + 1795102479.987, + "0" + ], + [ + 1795102484.987, + "0" + ], + [ + 1795102489.987, + "0" + ], + [ + 1795102494.987, + "0" + ], + [ + 1795102499.987, + "0" + ], + [ + 1795102504.987, + "0" + ], + [ + 1795102509.987, + "0" + ], + [ + 1795102514.987, + "0" + ], + [ + 1795102519.987, + "0" + ], + [ + 1795102524.987, + "0" + ], + [ + 1795102529.987, + "0" + ], + [ + 1795102534.987, + "0" + ], + [ + 1795102539.987, + "0" + ], + [ + 1795102544.987, + "0" + ], + [ + 1795102549.987, + "0" + ], + [ + 1795102554.987, + "0" + ], + [ + 1795102559.987, + "0" + ], + [ + 1795102564.987, + "0" + ], + [ + 1795102569.987, + "0" + ], + [ + 1795102574.987, + "0" + ], + [ + 1795102579.987, + "0" + ], + [ + 1795102584.987, + "0" + ], + [ + 1795102589.987, + "0" + ] + ] + } + ] + } + }, + "ready_bad": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "__name__": "lab6:ready_bad_ratio:2m" + }, + "values": [ + [ + 1795101974.987, + "0" + ], + [ + 1795101979.987, + "0" + ], + [ + 1795101984.987, + "0" + ], + [ + 1795101989.987, + "0" + ], + [ + 1795101994.987, + "0" + ], + [ + 1795101999.987, + "0" + ], + [ + 1795102004.987, + "0" + ], + [ + 1795102009.987, + "0" + ], + [ + 1795102014.987, + "0" + ], + [ + 1795102019.987, + "0" + ], + [ + 1795102024.987, + "0" + ], + [ + 1795102029.987, + "0" + ], + [ + 1795102034.987, + "0" + ], + [ + 1795102039.987, + "0" + ], + [ + 1795102044.987, + "0" + ], + [ + 1795102049.987, + "0" + ], + [ + 1795102054.987, + "0" + ], + [ + 1795102059.987, + "0" + ], + [ + 1795102064.987, + "0" + ], + [ + 1795102069.987, + "0" + ], + [ + 1795102074.987, + "0" + ], + [ + 1795102079.987, + "0" + ], + [ + 1795102084.987, + "0" + ], + [ + 1795102089.987, + "0" + ], + [ + 1795102094.987, + "0" + ], + [ + 1795102099.987, + "0" + ], + [ + 1795102104.987, + "0" + ], + [ + 1795102109.987, + "0" + ], + [ + 1795102114.987, + "0" + ], + [ + 1795102119.987, + "0" + ], + [ + 1795102124.987, + "0" + ], + [ + 1795102129.987, + "0" + ], + [ + 1795102134.987, + "0" + ], + [ + 1795102139.987, + "0" + ], + [ + 1795102144.987, + "0" + ], + [ + 1795102149.987, + "0" + ], + [ + 1795102154.987, + "0" + ], + [ + 1795102159.987, + "0" + ], + [ + 1795102164.987, + "0" + ], + [ + 1795102169.987, + "0" + ], + [ + 1795102174.987, + "0" + ], + [ + 1795102179.987, + "0" + ], + [ + 1795102184.987, + "0" + ], + [ + 1795102189.987, + "0" + ], + [ + 1795102194.987, + "0" + ], + [ + 1795102199.987, + "0" + ], + [ + 1795102204.987, + "0" + ], + [ + 1795102209.987, + "0" + ], + [ + 1795102214.987, + "0" + ], + [ + 1795102219.987, + "0" + ], + [ + 1795102224.987, + "0" + ], + [ + 1795102229.987, + "0" + ], + [ + 1795102234.987, + "0" + ], + [ + 1795102239.987, + "0" + ], + [ + 1795102244.987, + "0" + ], + [ + 1795102249.987, + "0" + ], + [ + 1795102254.987, + "0" + ], + [ + 1795102259.987, + "0" + ], + [ + 1795102264.987, + "0" + ], + [ + 1795102269.987, + "0" + ], + [ + 1795102274.987, + "0" + ], + [ + 1795102279.987, + "0" + ], + [ + 1795102284.987, + "0" + ], + [ + 1795102289.987, + "0" + ], + [ + 1795102294.987, + "0" + ], + [ + 1795102299.987, + "0" + ], + [ + 1795102304.987, + "0" + ], + [ + 1795102309.987, + "0" + ], + [ + 1795102314.987, + "0" + ], + [ + 1795102319.987, + "0" + ], + [ + 1795102324.987, + "0" + ], + [ + 1795102329.987, + "0" + ], + [ + 1795102334.987, + "0" + ], + [ + 1795102339.987, + "0" + ], + [ + 1795102344.987, + "0" + ], + [ + 1795102349.987, + "0" + ], + [ + 1795102354.987, + "0" + ], + [ + 1795102359.987, + "0" + ], + [ + 1795102364.987, + "0" + ], + [ + 1795102369.987, + "0" + ], + [ + 1795102374.987, + "0" + ], + [ + 1795102379.987, + "0" + ], + [ + 1795102384.987, + "0" + ], + [ + 1795102389.987, + "0" + ], + [ + 1795102394.987, + "0.017241379310344827" + ], + [ + 1795102399.987, + "0.06779661016949153" + ], + [ + 1795102404.987, + "0.11864406779661017" + ], + [ + 1795102409.987, + "0.15254237288135594" + ], + [ + 1795102414.987, + "0.2033898305084746" + ], + [ + 1795102419.987, + "0.2105263157894737" + ], + [ + 1795102424.987, + "0.25" + ], + [ + 1795102429.987, + "0.30357142857142855" + ], + [ + 1795102434.987, + "0.33928571428571425" + ], + [ + 1795102439.987, + "0.39285714285714285" + ], + [ + 1795102444.987, + "0.42857142857142855" + ], + [ + 1795102449.987, + "0.4821428571428572" + ], + [ + 1795102454.987, + "0.5263157894736842" + ], + [ + 1795102459.987, + "0.5714285714285714" + ], + [ + 1795102464.987, + "0.6140350877192983" + ], + [ + 1795102469.987, + "0.6607142857142857" + ], + [ + 1795102474.987, + "0.7142857142857143" + ], + [ + 1795102479.987, + "0.75" + ], + [ + 1795102484.987, + "0.8035714285714286" + ], + [ + 1795102489.987, + "0.8421052631578948" + ], + [ + 1795102494.987, + "0.8928571428571428" + ], + [ + 1795102499.987, + "0.894736842105263" + ], + [ + 1795102504.987, + "0.9107142857142857" + ], + [ + 1795102509.987, + "0.8771929824561403" + ], + [ + 1795102514.987, + "0.8392857142857143" + ], + [ + 1795102519.987, + "0.7857142857142857" + ], + [ + 1795102524.987, + "0.75" + ], + [ + 1795102529.987, + "0.6964285714285714" + ], + [ + 1795102534.987, + "0.6724137931034483" + ], + [ + 1795102539.987, + "0.6271186440677966" + ], + [ + 1795102544.987, + "0.576271186440678" + ], + [ + 1795102549.987, + "0.5423728813559322" + ], + [ + 1795102554.987, + "0.4915254237288136" + ], + [ + 1795102559.987, + "0.4576271186440678" + ], + [ + 1795102564.987, + "0.4067796610169492" + ], + [ + 1795102569.987, + "0.3620689655172414" + ], + [ + 1795102574.987, + "0.3220338983050848" + ], + [ + 1795102579.987, + "0.2711864406779661" + ], + [ + 1795102584.987, + "0.23728813559322037" + ], + [ + 1795102589.987, + "0.1864406779661017" + ] + ] + } + ] + } + }, + "alerts": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "__name__": "ALERTS", + "alertname": "APIAvailabilityFastBurn", + "alertstate": "firing", + "severity": "critical", + "slo": "api" + }, + "values": [ + [ + 1795102194.987, + "1" + ], + [ + 1795102199.987, + "1" + ], + [ + 1795102204.987, + "1" + ], + [ + 1795102209.987, + "1" + ], + [ + 1795102214.987, + "1" + ], + [ + 1795102219.987, + "1" + ], + [ + 1795102224.987, + "1" + ], + [ + 1795102229.987, + "1" + ], + [ + 1795102234.987, + "1" + ], + [ + 1795102239.987, + "1" + ], + [ + 1795102244.987, + "1" + ], + [ + 1795102249.987, + "1" + ], + [ + 1795102254.987, + "1" + ], + [ + 1795102259.987, + "1" + ], + [ + 1795102264.987, + "1" + ], + [ + 1795102269.987, + "1" + ], + [ + 1795102274.987, + "1" + ], + [ + 1795102279.987, + "1" + ] + ] + }, + { + "metric": { + "__name__": "ALERTS", + "alertname": "APIAvailabilitySlowBurn", + "alertstate": "firing", + "severity": "warning", + "slo": "api" + }, + "values": [ + [ + 1795102219.987, + "1" + ], + [ + 1795102224.987, + "1" + ], + [ + 1795102229.987, + "1" + ], + [ + 1795102234.987, + "1" + ], + [ + 1795102239.987, + "1" + ], + [ + 1795102244.987, + "1" + ], + [ + 1795102249.987, + "1" + ], + [ + 1795102254.987, + "1" + ], + [ + 1795102259.987, + "1" + ], + [ + 1795102264.987, + "1" + ], + [ + 1795102269.987, + "1" + ], + [ + 1795102274.987, + "1" + ], + [ + 1795102279.987, + "1" + ], + [ + 1795102284.987, + "1" + ], + [ + 1795102289.987, + "1" + ], + [ + 1795102294.987, + "1" + ], + [ + 1795102299.987, + "1" + ], + [ + 1795102304.987, + "1" + ], + [ + 1795102309.987, + "1" + ] + ] + }, + { + "metric": { + "__name__": "ALERTS", + "alertname": "APIHighErrorRate", + "alertstate": "firing", + "severity": "warning" + }, + "values": [ + [ + 1795102114.987, + "1" + ], + [ + 1795102119.987, + "1" + ], + [ + 1795102124.987, + "1" + ], + [ + 1795102129.987, + "1" + ], + [ + 1795102179.987, + "1" + ], + [ + 1795102184.987, + "1" + ], + [ + 1795102189.987, + "1" + ], + [ + 1795102194.987, + "1" + ], + [ + 1795102199.987, + "1" + ], + [ + 1795102204.987, + "1" + ], + [ + 1795102209.987, + "1" + ], + [ + 1795102214.987, + "1" + ], + [ + 1795102219.987, + "1" + ], + [ + 1795102224.987, + "1" + ], + [ + 1795102229.987, + "1" + ], + [ + 1795102234.987, + "1" + ], + [ + 1795102239.987, + "1" + ], + [ + 1795102244.987, + "1" + ], + [ + 1795102249.987, + "1" + ], + [ + 1795102254.987, + "1" + ], + [ + 1795102259.987, + "1" + ], + [ + 1795102264.987, + "1" + ], + [ + 1795102269.987, + "1" + ], + [ + 1795102274.987, + "1" + ], + [ + 1795102279.987, + "1" + ] + ] + }, + { + "metric": { + "__name__": "ALERTS", + "alertname": "AttachmentQueueLag", + "alertstate": "firing", + "severity": "warning" + }, + "values": [ + [ + 1795102394.987, + "1" + ], + [ + 1795102399.987, + "1" + ], + [ + 1795102404.987, + "1" + ], + [ + 1795102409.987, + "1" + ], + [ + 1795102414.987, + "1" + ], + [ + 1795102419.987, + "1" + ], + [ + 1795102424.987, + "1" + ], + [ + 1795102429.987, + "1" + ], + [ + 1795102434.987, + "1" + ], + [ + 1795102439.987, + "1" + ], + [ + 1795102444.987, + "1" + ], + [ + 1795102449.987, + "1" + ], + [ + 1795102454.987, + "1" + ], + [ + 1795102459.987, + "1" + ], + [ + 1795102464.987, + "1" + ], + [ + 1795102469.987, + "1" + ], + [ + 1795102474.987, + "1" + ], + [ + 1795102479.987, + "1" + ], + [ + 1795102484.987, + "1" + ], + [ + 1795102489.987, + "1" + ], + [ + 1795102494.987, + "1" + ] + ] + }, + { + "metric": { + "__name__": "ALERTS", + "alertname": "AttachmentReadyFastBurn", + "alertstate": "firing", + "severity": "critical", + "slo": "ready" + }, + "values": [ + [ + 1795102444.987, + "1" + ], + [ + 1795102449.987, + "1" + ], + [ + 1795102454.987, + "1" + ], + [ + 1795102459.987, + "1" + ], + [ + 1795102464.987, + "1" + ], + [ + 1795102469.987, + "1" + ], + [ + 1795102474.987, + "1" + ], + [ + 1795102479.987, + "1" + ], + [ + 1795102484.987, + "1" + ], + [ + 1795102489.987, + "1" + ], + [ + 1795102494.987, + "1" + ], + [ + 1795102499.987, + "1" + ], + [ + 1795102504.987, + "1" + ], + [ + 1795102509.987, + "1" + ] + ] + }, + { + "metric": { + "__name__": "ALERTS", + "alertname": "AttachmentReadySlowBurn", + "alertstate": "firing", + "severity": "warning", + "slo": "ready" + }, + "values": [ + [ + 1795102479.987, + "1" + ], + [ + 1795102484.987, + "1" + ], + [ + 1795102489.987, + "1" + ], + [ + 1795102494.987, + "1" + ], + [ + 1795102499.987, + "1" + ], + [ + 1795102504.987, + "1" + ], + [ + 1795102509.987, + "1" + ], + [ + 1795102514.987, + "1" + ], + [ + 1795102519.987, + "1" + ], + [ + 1795102524.987, + "1" + ], + [ + 1795102529.987, + "1" + ], + [ + 1795102534.987, + "1" + ], + [ + 1795102539.987, + "1" + ] + ] + } + ] + } + } +} diff --git a/evidence/lab6-source-manifest.json b/evidence/lab6-source-manifest.json new file mode 100644 index 0000000..8f7f56b --- /dev/null +++ b/evidence/lab6-source-manifest.json @@ -0,0 +1,130 @@ +{ + "base_commit": "470d9d6f1e4fe4f594e37c2ba59550ee5ca54fc1", + "working_tree": true, + "files": { + ".dockerignore": "a092ac618bd90f555f48c85fc58cad3ac9f51f4de4379b242f51e14e3d468b38", + ".env.example": "690df84f2c2f7200bd05a6ae325277f0771df9c57e581003746e2c20e8119c05", + ".github/pull_request_template.lab5.md": "e3e9afa285870e2133b3dbc30fec881bae7418605789bdb1eaf497a2eb62914a", + ".github/pull_request_template.md": "fd1a90691c2197f05c4aa0dd43aea9eda4be1e6abb0e1bb1430991228eda8e64", + ".github/workflows/application.yml": "e9b1309023f630f7773c7bb7c525e68662245c1657e5b773a9393ec50d0d88a5", + ".github/workflows/contracts.yml": "a45d7932ef14130aff8020bce79f043cccaf9692d697088a754d309a03f7d3e8", + ".gitignore": "c26ab45b3a778a5376f6db4f624a51c5fdbf1a404f9a26df8613449277d95438", + "CONTRIBUTING.md": "2c01dd698cc9653dece49a94c9fd3f58d36e84764aaa643b46e26d7e599dc12e", + "COURSE.md": "c34d53cd082d194aa4b58f504c4e0348d25b6afec3d6a9c0963dd01c9c81e971", + "Dockerfile": "e2a683428301e5bf584fb0c067b447c22e2a088ff13ce147b7f7b8b2261cd8ea", + "Dockerfile.example": "fd7ac9491c0c3c663b083c41a762fb38bc2d212fa4a2ed3e6a0702af72108709", + "MAINTAINERS.md": "f2e7e9d59e06f260ccf3acc9baf79c4a3b66e1bc370ff6e21cbefb338210630a", + "Makefile": "c05bd07058ae0949dd4905afdae7674bc7e3e63863a6cdf8b72661140d445826", + "README.md": "583fa7625d728ead8a6ee2cd2de9f117ed3f4a1d9696c0bb9fb602fe549a31db", + "REPORT.lab5.md": "ce77ed0e6cc43a6283a9a2964a2508375496b274996222ae1ab5045b508f1d5f", + "REPORT.md": "4107c5e486c13d8e48eb0f2d37bbdabe267d857af56a17fe9ac879b5b652a6ea", + "compose.example.yaml": "9bfd9eacae286dd9bffe29d6fe349534552560f966c3c3e6e6540e92d7357afb", + "compose.yaml": "aed7a12a1ceb68bc729a102b7d76ab9354d426a136245921c37556f20a06a79c", + "contracts/README.md": "72df27a559f88ffa3fd8c3c79d3c32f6db2c5986a0602677c51994c434a3fc4f", + "contracts/attachment-states.md": "cf0519e137410e4b34ab632aadb8ad8a381d4b73033e27b354c58eeacbdf86d6", + "contracts/examples.json": "381cc18c03789e935b391d007fb5843964be9cfd9f7530cb6b1996ed4a1c4c37", + "contracts/infrastructure.json": "45264a3db653bb79de8d8fcb8d70733924f3cbfcba0fa1253cecbc6828bce217", + "contracts/jobs.schema.json": "9ccf9b11466c0c3a26a941520ac5ff543241fc4935e3a6eebb01ef303779b10d", + "contracts/observability.md": "882c120c0e153a06c9cc824a802e317c58eb6590bc3d3a576f2e0a96f8597b62", + "contracts/openapi.json": "2c4074e8fac6bed6337fbbc0aaed0ae2321a76ef30ea68afad80a1f39699535a", + "docs/runbook.md": "5b9b9529ecce1f0c510e9463783db3d5d0dd14f709d2a2d571c0094714067b49", + "docs/schema.md": "205d9a1524343b602f87b14f64ebc207372c747c1e38a3222d1c733c002093ec", + "examples/requests.http": "8c086fd540092b3e31a945a416632bcda6c4bea39253f86b076f791b362cf5e4", + "infra/README.md": "0e804fa51debba4c38d3cc84abf4704897665e0335b272060363f03aadd6354f", + "infra/grafana/dashboards/messenger.json": "ed102001c4002a3e63617216571979727c05b10866a6eb58c4ac795110402510", + "infra/grafana/provisioning/dashboards/provider.yaml": "d7aa7568978a55168509eb23f2522d081627cf3c1e141a356b2c6d8e3ff86f62", + "infra/grafana/provisioning/datasources/sources.yaml": "b89e3fa70a4432fac952a2bb7a55a0dbb6cae92787090a8d1b30f3468db56d82", + "infra/haproxy.cfg": "c80c30717494c8185eda0aa0437734e3b011bff3d213ca2fa2b2e0af83c88600", + "infra/otel/README.md": "0d6c93589f7b96b7045c1b82a261362697d25f14c9e1a4946b8ea8a977c1e842", + "infra/otel/alerts.yaml": "e7a1223c64b7040cbd76b754e19ed17a6d321aaf6da1996dafcd608b3ca669af", + "infra/otel/collector.yaml": "92c759d393d9669815131acca5c8df7ed1c71dc7c7818a7b445cefcb8c040379", + "infra/otel/loki.yaml": "a1b1e6af51a0f34a7b8356d6eab523b855731b292e129cbe6a480fe0551c2eeb", + "infra/otel/prometheus.yaml": "df8340a67cb44830d1d1a6c19b87acb22d00b8b85c2993ab37a28599b43829ff", + "infra/otel/tempo.yaml": "b7d437347fe6b344de2e6499426931330c6d41ce1ba2f39492e03069e2908a43", + "lab.json": "807dbb02b180ddb13dc898c117e09495e15a462e83f4b55133ba174f089594ce", + "migrations/001_initial.sql": "a319949c962c54c5ba537a284ff38f77f2ba04c5bbc748873889d49c2bf48a46", + "migrations/002_message_idempotency.sql": "8e484326b7861b8acf4b847d7126639bf6216e635012e1e969006e86caf902ed", + "migrations/003_attachments.sql": "9deda173c984675092869022715c32648aa7b91cf8f8641bb5daf68c6f871aba", + "migrations/004_attachment_jobs.sql": "d88a95701d0b7359a731fa81aff22b5b3e1948fb29aacdf1e7e27b9136d27427", + "migrations/005_observability.sql": "fb23df863aedb6686c185d59a2499134c09c2e6ad9859c2b7e1648027aa815ea", + "requirements-dev.txt": "4cefdb67007caf026d1613be7c519c01b4c8343db7c4f4b618677e01cf113525", + "requirements.txt": "861b5ad246e5cfa706a7cdcf7d3a4a9e9feca3261d89b387e1bb553244339ed6", + "research/README.lab2.md": "59606b7a6325ac75749963f45b6b367c93d5a3a97093cb8850587ee9ae92b5da", + "research/README.md": "cd14eeb3cfdb81a70269f4503f4e91383c99e482757bb462f9120f684c6dcca7", + "research/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "research/benchmark.py": "d66405d705f6941735efb7f297df2539ec73a31f3efa989723e4ed02f72af721", + "research/observations.py": "238b8da50a1c48656daf2fe424e60d0de46ae5a4743c17ac17dda064b555a7f6", + "research/run.py": "843a33bb642ebe0648ab8ccf551adfb2e05d9fe1127d8cbd83bf762f872b243b", + "research/slo.py": "7bf6848d94e483f65598681d39b908635d205893c89874fcbe14c229de8ca233", + "scripts/build.sh": "8f67493f0563b0dce0e0f327655c4838a21c04d9d887a03820798f4f56518c2e", + "scripts/check-docker.sh": "43e6d221f386af27eec2b49a18ed6c48c284331c7a06dcb5ee03e189e0bca3ba", + "scripts/check-lab6.sh": "eb8b12eb0b770a5503ffeb349cd3b79b2777e6c7b5bb6a41186d14ef3b1f2d7a", + "scripts/check.sh": "c0de27e3d079b503f6cf2a1606bc6d156b5ac35c61455582bab3d5ee809a17a6", + "scripts/check_contract.py": "7c4402c2a6ab80d3b4676e1866d6ecc87b6daea14c080bcb9be9a7e8b959d671", + "scripts/demo.sh": "8363935669d34bf56f1e170261d590c75dce9470308da90f6e700034a9bd01f3", + "scripts/init_local.py": "279f03cc5a4bc1735396594ec238c3946bc97a7343baf93f94e6abbe12f43e1f", + "scripts/migrate.sh": "8700e123e7f154b2b80926c83ffe384cd39572bb3e063d807dfa3c144c135913", + "scripts/replicas.py": "fa2700c2a486c5919cfa1095068307ed445a6268d8dfcf5b75ce15a1629509eb", + "scripts/research-lab2.sh": "081ea2b07c338b03e08eaba8316c1de4c9b49e0d83bdded7b4de956dc77352dc", + "scripts/research.sh": "84f34fcb0cf8338d8ff8749524db68aab0e2c7b328e2f4b45ad2cf980d179a66", + "scripts/restart.example.sh": "0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b", + "scripts/restart.sh": "6fd3eb1e0e65df0f600d88cccce52068d60ea1f58a8a125b87622ec62b82a5de", + "scripts/rolling-update.sh": "dc09335e3e8989b42bbac75a9aec9e722ab234541361830bd924f544b45f489d", + "scripts/run-local.sh": "61d5021aaca500ae8ac4249d6d63a839edbd811df006a56ea27a916f9fcbeb71", + "scripts/start-worker.sh": "b5e8df907b645ab2d81799933f9e94cf160f4922ebdba21b9d1e52ee59c8d93f", + "scripts/start.sh": "5314c8d051541b330510d6f59d6aefebd5d3ee020464e14dd9f46c09af2f6c61", + "scripts/stop-one.example.sh": "1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811", + "scripts/stop-one.sh": "fd6e58a604aaed3833c220f4e828ce52bd2c61a905a232a0c50da74b6a3b2edb", + "scripts/stop-worker.sh": "32ef73cfc0ef91e5acd437934a46e4567d83ab926247bf624be5aeb4ce3d3993", + "scripts/stop.sh": "db7ef5f1723566859b211f02f753c65657e6820f8135a4bc07ce6c544284df0d", + "scripts/validate_openapi.py": "4e981acbfc10dac58817bf922d5896e996798551fedc1b2385130e1cddb13d54", + "scripts/wait-observability.py": "af89dbdf9c0e4d9ae11bb7b80143fd8af08b254578159e695e660c55345693c7", + "scripts/wait-ready.sh": "c24e7a2297bbcbeb69451d17875a669b61a0b6b86e9a03fb33dabb2daea0dce7", + "src/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "src/__main__.py": "91819b838eef82165a9ef67aaf45cbb7a54d131f170076e1dde672d9a77e121b", + "src/app.py": "796aee198f9aa934bac388aa05e1cc2b067643db5f1bd2a843cc0618e1b9b09d", + "src/attachments.py": "9d4c4d3f76e489dfe3b96d89f892f7b9e9222e4ffec667d419df7dcc5850e7d5", + "src/auth.py": "75ad98d17940efb2e4f6dfc584b371a4edc6c397af4fb46f3666ee9e2d043866", + "src/bootstrap_s3.py": "ad8330225e4697873d2961f874cbdfbb69b66301c6d0ccf28d6999cc5b9c8191", + "src/broker.py": "13c5c025338ccd90eb74e48d8892aba4547adc33b71807fad021a99eb37d795e", + "src/cleanup.py": "839162ed4bf01db7734025b3b14e21a30449e2b7e5b6c3d60b39c9bd723c98b5", + "src/config.py": "ad9d1f94e796b82a7a7b9b93a98d94ca588decf02a65e588b34cab362ef0a6a0", + "src/db.py": "803446c82ebbfa11b33ef41c100b4b06204f7ba3fffcec8092e0e91f6b992882", + "src/errors.py": "f020bf3e5c9619c6ef47e1c8910a19e3331140eb9dd2bba74981e89e855e1149", + "src/faults.py": "b1ca87cf1c75a050c8572c9a3b05b569d2e459c7c5d4ce23efefbeffd2774e6b", + "src/healthcheck.py": "bb8c06c1836f601f77755de3aca4b1092294784d3c33d60bd27b41778cd130be", + "src/image_pipeline.py": "f1800c26913612739f919b6880e5b673e986c198cb4ea9509e105240f40f1fa8", + "src/job_admin.py": "51781bdeacf644c36118a9972eab125893d6bf65a3eaa1143d8eb3f3d24631e1", + "src/jobs.py": "9bbed284d1f61355d65e84271191242989b10f97bb803792907444908ea15165", + "src/migrate.py": "66b66b9d244925957ee8559dc734e872172da403c688c2bf20ae8312d7c3b260", + "src/models.py": "6f9adcc886e05c7c28c540393ca4a3d22083693f30e33a51a074306f511acc15", + "src/object_store.py": "177a9cd69e0a4f494a1cdafc92544cf4a0f150842a0b7d5e826b72c13b49532a", + "src/pagination.py": "ca07be71f21ba56873d77c9422530a7a5050679b57c055b88fcf547efc112558", + "src/passwords.py": "9cb66e8e841faee89868af041a793b0b511df49b85b8d5b1a27ca296165bf1de", + "src/publisher.py": "2f99db91efdf3f7e06383954fe54c30745af402efd07052bd372ed8e1ab78211", + "src/storage.py": "58c49912188944d739d8ebc9a0d834958b605b56f9e6296d59e9b99c2735961f", + "src/telemetry.py": "3151acbe5eed23c92cc2bf2a644bc0035a12be34fdec99737042487c1ef87af5", + "src/worker.py": "1215386583876119eb817934461b56edc5416369282af017b33e4a6a9d79da0a", + "student-tests/attachment_checks.py": "219648a7ea58d81ed2c709c1961d382558654c230db369f320d4e39f6b5ba4fb", + "student-tests/docker_lifecycle.py": "06a49931d151419eea5f78668a737bc17a5d31f0559d78e5429be882fd10046c", + "student-tests/helpers.py": "3723976953cb1212e6773e8fc9771148232e4d95ff665f049e5f47d23a97d7ba", + "student-tests/lab5_pipeline.py": "c23d50767759df0ea6642bff7a56b2f3cd0eb1010dd77b42cd941e15ca3650d3", + "student-tests/lab6_observability.py": "1a16f9e7315f980089dfb82c45e009ad3979c9754dd305b2628fe16c36e68b6c", + "student-tests/postgres_pause.py": "e81dee40fe1356940b7b8a65e2c9cf5cc1510adc6bf831dd235041044f654548", + "student-tests/promtool-slo.yaml": "f265001af6026fadc0677132a510f8dccf8ff1b5e1590ea156efec571040779e", + "student-tests/test_api.py": "5a60014b01ebc40a14c037ded532995ebb680c0aa5923a910460a8da181fc32f", + "student-tests/test_attachments.py": "8660c45842e9987b814f7075f3ae23c446c156cb7b755dd2734fed0b5b11af56", + "student-tests/test_concurrency.py": "ae48809fb1acef260abb04ba3598475a9456908d3193c90445458837547a9d84", + "student-tests/test_idempotency.py": "1f34c297029b182a0f66b99f007bc1dfebf7e6e7ca6fb5d695d0a8be37135d32", + "student-tests/test_sessions.py": "4d8fe299727266bf1ca1d7fd25b8ee3dd1780b10cce23e0af48bd47da95a852e", + "student-tests/verify_restart.py": "5c7d5197ede3d92351e07cc6768a7acc3ce204120e21f744ca48a98594def33f", + "tests/README.md": "e0bfa7e196797f6a5bf6e44d8ad2ffeb82651927ff5d8484eb1c2dda1ec7dec5", + "tests/client.py": "8d50976fc7c7bbb81e88a152f613c61ab25eaf53dba33c6a9dfab19f868d546d", + "tests/fixtures/README.md": "c001783760b4f87a7a8441a7ba55f5d080ff48e1397d3581d72a88d7691f96c2", + "tests/fixtures/sample.png": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "tests/image_probe.py": "a9a4f89181c8e0fda6ce42e1a1af32fb42d8e3ae519e314207127c9b9b448b13", + "tests/load.py": "6d3511ea719c7a7e841083e73d435c86dcbfd9f4e5c2a07d62eb9614b01d9933", + "tests/resilience.py": "1f96663a174b3978d24eff66f2741f5f8c9b788d3242261d37a972720c52788b", + "tests/smoke.py": "52ac38eb343ff7d43e7543588e1da58ca06c9fce74b2b0d0bf0e4982bc23c1db" + } +} diff --git a/evidence/lab6-telemetry-persistence.json b/evidence/lab6-telemetry-persistence.json new file mode 100644 index 0000000..c9c44d8 --- /dev/null +++ b/evidence/lab6-telemetry-persistence.json @@ -0,0 +1,12 @@ +{ + "passed": true, + "trace_id": "93e1177f9568f5bbfc0b6967a1324a84", + "stored_span_count": 59, + "after_full_restart": true, + "prometheus_series": 4, + "datasources": [ + "loki", + "prometheus", + "tempo" + ] +} diff --git a/evidence/lab6-tempo-search.json b/evidence/lab6-tempo-search.json new file mode 100644 index 0000000..796344f --- /dev/null +++ b/evidence/lab6-tempo-search.json @@ -0,0 +1,2167 @@ +{ + "traces": [ + { + "traceID": "93e1177f9568f5bbfc0b6967a1324a84", + "rootServiceName": "", + "startTimeUnixNano": "1795100977539356419", + "durationMs": 1683, + "spanSet": { + "spans": [ + { + "spanID": "a174323f5d58b077", + "startTimeUnixNano": "1795100977539545960", + "durationNanos": "162334", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "af89685897ba3254", + "startTimeUnixNano": "1795100977539772127", + "durationNanos": "864917", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "8e5d47d485872806", + "startTimeUnixNano": "1795100977540718919", + "durationNanos": "355291", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 35 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "a174323f5d58b077", + "startTimeUnixNano": "1795100977539545960", + "durationNanos": "162334", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "af89685897ba3254", + "startTimeUnixNano": "1795100977539772127", + "durationNanos": "864917", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "8e5d47d485872806", + "startTimeUnixNano": "1795100977540718919", + "durationNanos": "355291", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 35 + } + ], + "serviceStats": { + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 35, + "errorCount": 1 + } + } + }, + { + "traceID": "3bf979b7cffce8862b9359032b60533d", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100931652092009", + "durationMs": 363, + "spanSet": { + "spans": [ + { + "spanID": "e86e3e30d585e9d5", + "startTimeUnixNano": "1795100931664447301", + "durationNanos": "120916", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "d0efbd02272abb93", + "startTimeUnixNano": "1795100931664643676", + "durationNanos": "463500", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "3765483886d8b1ef", + "startTimeUnixNano": "1795100931665198384", + "durationNanos": "205167", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "e86e3e30d585e9d5", + "startTimeUnixNano": "1795100931664447301", + "durationNanos": "120916", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "d0efbd02272abb93", + "startTimeUnixNano": "1795100931664643676", + "durationNanos": "463500", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "3765483886d8b1ef", + "startTimeUnixNano": "1795100931665198384", + "durationNanos": "205167", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 22 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "689e3e6386f5ae28f7127bc5741a66fd", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100931002158884", + "durationMs": 571, + "spanSet": { + "spans": [ + { + "spanID": "86ab6452b241921b", + "startTimeUnixNano": "1795100931187937800", + "durationNanos": "124959", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "18e44c021922476e", + "startTimeUnixNano": "1795100931188129467", + "durationNanos": "325667", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "922a20f4e1beabb4", + "startTimeUnixNano": "1795100931188532384", + "durationNanos": "319916", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "86ab6452b241921b", + "startTimeUnixNano": "1795100931187937800", + "durationNanos": "124959", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "18e44c021922476e", + "startTimeUnixNano": "1795100931188129467", + "durationNanos": "325667", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "922a20f4e1beabb4", + "startTimeUnixNano": "1795100931188532384", + "durationNanos": "319916", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 22 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "10d559185817ab2c73b2c9433545376b", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100930534806175", + "durationMs": 722, + "spanSet": { + "spans": [ + { + "spanID": "c411fda3deb2c96a", + "startTimeUnixNano": "1795100930813108342", + "durationNanos": "143458", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "c5625111e1b6ab4f", + "startTimeUnixNano": "1795100930813318175", + "durationNanos": "745167", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "ba00ceb88ceb8b71", + "startTimeUnixNano": "1795100930814560092", + "durationNanos": "398708", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "c411fda3deb2c96a", + "startTimeUnixNano": "1795100930813108342", + "durationNanos": "143458", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "c5625111e1b6ab4f", + "startTimeUnixNano": "1795100930813318175", + "durationNanos": "745167", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "ba00ceb88ceb8b71", + "startTimeUnixNano": "1795100930814560092", + "durationNanos": "398708", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 22 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "6fafddd0fc061cc115ee42b6c84461a9", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100930105112925", + "durationMs": 1081, + "spanSet": { + "spans": [ + { + "spanID": "b48f329f4731abf4", + "startTimeUnixNano": "1795100930744300842", + "durationNanos": "273041", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "01a845ebd9c2e50a", + "startTimeUnixNano": "1795100930744688008", + "durationNanos": "1947209", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "773d62d626cd603b", + "startTimeUnixNano": "1795100930746780133", + "durationNanos": "295834", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "b48f329f4731abf4", + "startTimeUnixNano": "1795100930744300842", + "durationNanos": "273041", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "01a845ebd9c2e50a", + "startTimeUnixNano": "1795100930744688008", + "durationNanos": "1947209", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "773d62d626cd603b", + "startTimeUnixNano": "1795100930746780133", + "durationNanos": "295834", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 22 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "9dd6c4ed951f9ef877cd8271375692fe", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100930062560383", + "durationMs": 747, + "spanSet": { + "spans": [ + { + "spanID": "990b2d1a2039cda3", + "startTimeUnixNano": "1795100930131613842", + "durationNanos": "1705208", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "c3eb26ce4bcdc539", + "startTimeUnixNano": "1795100930134750383", + "durationNanos": "4041792", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "125d52439be43832", + "startTimeUnixNano": "1795100930139292383", + "durationNanos": "2065959", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "990b2d1a2039cda3", + "startTimeUnixNano": "1795100930131613842", + "durationNanos": "1705208", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "c3eb26ce4bcdc539", + "startTimeUnixNano": "1795100930134750383", + "durationNanos": "4041792", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "125d52439be43832", + "startTimeUnixNano": "1795100930139292383", + "durationNanos": "2065959", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 22 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "db8eac21f8d69fa9729e4d75a2d7ae5d", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100929934993341", + "durationMs": 806, + "spanSet": { + "spans": [ + { + "spanID": "ec6835efe2f9dce0", + "startTimeUnixNano": "1795100930119139633", + "durationNanos": "401792", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "450250348ecd9bfa", + "startTimeUnixNano": "1795100930119958133", + "durationNanos": "1112667", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "ef4ead74bf70fdfe", + "startTimeUnixNano": "1795100930121636133", + "durationNanos": "674125", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "ec6835efe2f9dce0", + "startTimeUnixNano": "1795100930119139633", + "durationNanos": "401792", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "450250348ecd9bfa", + "startTimeUnixNano": "1795100930119958133", + "durationNanos": "1112667", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "ef4ead74bf70fdfe", + "startTimeUnixNano": "1795100930121636133", + "durationNanos": "674125", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 22 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "e35b5fc0b69995b9149051757ea8282b", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100929189140674", + "durationMs": 607, + "spanSet": { + "spans": [ + { + "spanID": "b33b2dea2ef3cf3e", + "startTimeUnixNano": "1795100929352659674", + "durationNanos": "152292", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "520f06bd761bfea6", + "startTimeUnixNano": "1795100929352905008", + "durationNanos": "750958", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "9f73cd93b380775e", + "startTimeUnixNano": "1795100929353914341", + "durationNanos": "489625", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "b33b2dea2ef3cf3e", + "startTimeUnixNano": "1795100929352659674", + "durationNanos": "152292", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "520f06bd761bfea6", + "startTimeUnixNano": "1795100929352905008", + "durationNanos": "750958", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "9f73cd93b380775e", + "startTimeUnixNano": "1795100929353914341", + "durationNanos": "489625", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 22 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "1e17913818553dcbe8d1c461906b2c9d", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100928393494591", + "durationMs": 594, + "spanSet": { + "spans": [ + { + "spanID": "ad95ce5e7e8f268c", + "startTimeUnixNano": "1795100928589569216", + "durationNanos": "398330833", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "50b6b8acdde573cb", + "startTimeUnixNano": "1795100928589689716", + "durationNanos": "1664750", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "92b1b76767fcb568", + "startTimeUnixNano": "1795100928589820091", + "durationNanos": "211416", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "ad95ce5e7e8f268c", + "startTimeUnixNano": "1795100928589569216", + "durationNanos": "398330833", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "50b6b8acdde573cb", + "startTimeUnixNano": "1795100928589689716", + "durationNanos": "1664750", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "92b1b76767fcb568", + "startTimeUnixNano": "1795100928589820091", + "durationNanos": "211416", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 22 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "ee75cb26e4867a367d38bdbb3d78062b", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100925228057547", + "durationMs": 471, + "spanSet": { + "spans": [ + { + "spanID": "2ef62e4f624d9437", + "startTimeUnixNano": "1795100925306206298", + "durationNanos": "286333", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "813756e5775d439a", + "startTimeUnixNano": "1795100925306585964", + "durationNanos": "1061667", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "9b71d1416caa88ac", + "startTimeUnixNano": "1795100925307740214", + "durationNanos": "306375", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "2ef62e4f624d9437", + "startTimeUnixNano": "1795100925306206298", + "durationNanos": "286333", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "813756e5775d439a", + "startTimeUnixNano": "1795100925306585964", + "durationNanos": "1061667", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "9b71d1416caa88ac", + "startTimeUnixNano": "1795100925307740214", + "durationNanos": "306375", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 22 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "a0dfe17199730c4348c5d347853db610", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100593861945839", + "durationMs": 4148, + "spanSet": { + "spans": [ + { + "spanID": "28987133d0c755a2", + "startTimeUnixNano": "1795100596561870131", + "durationNanos": "147917", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "5ba195671015ca0d", + "startTimeUnixNano": "1795100596562060965", + "durationNanos": "954583", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "72dd65ea1e4342dc", + "startTimeUnixNano": "1795100596563076215", + "durationNanos": "394875", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 35 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "28987133d0c755a2", + "startTimeUnixNano": "1795100596561870131", + "durationNanos": "147917", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "5ba195671015ca0d", + "startTimeUnixNano": "1795100596562060965", + "durationNanos": "954583", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "72dd65ea1e4342dc", + "startTimeUnixNano": "1795100596563076215", + "durationNanos": "394875", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 35 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 25 + }, + "messenger-publisher": { + "spanCount": 2 + }, + "messenger-worker": { + "spanCount": 35, + "errorCount": 1 + } + } + }, + { + "traceID": "678a5464773b44dde0f62b6de25c98cf", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100493942855209", + "durationMs": 388, + "spanSet": { + "spans": [ + { + "spanID": "669c8810111a82bf", + "startTimeUnixNano": "1795100493993590750", + "durationNanos": "132959", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "0a81ca61c1e54e23", + "startTimeUnixNano": "1795100493993798959", + "durationNanos": "323125", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "0121cdf33933eff5", + "startTimeUnixNano": "1795100493994195500", + "durationNanos": "205875", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "669c8810111a82bf", + "startTimeUnixNano": "1795100493993590750", + "durationNanos": "132959", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "0a81ca61c1e54e23", + "startTimeUnixNano": "1795100493993798959", + "durationNanos": "323125", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "0121cdf33933eff5", + "startTimeUnixNano": "1795100493994195500", + "durationNanos": "205875", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 25 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "36a6a4e76c96370864c5ac8363423bc5", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100493305817292", + "durationMs": 533, + "spanSet": { + "spans": [ + { + "spanID": "eb3ea05e4b5eddf2", + "startTimeUnixNano": "1795100493499238292", + "durationNanos": "83958", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "3bfcd4ccd2bc019c", + "startTimeUnixNano": "1795100493499352334", + "durationNanos": "236750", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "bb4cdbaba94ba00a", + "startTimeUnixNano": "1795100493499634875", + "durationNanos": "169084", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "eb3ea05e4b5eddf2", + "startTimeUnixNano": "1795100493499238292", + "durationNanos": "83958", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "3bfcd4ccd2bc019c", + "startTimeUnixNano": "1795100493499352334", + "durationNanos": "236750", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "bb4cdbaba94ba00a", + "startTimeUnixNano": "1795100493499634875", + "durationNanos": "169084", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 25 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "f78f39957f7fc059ea184383bc2aa35c", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100493014607583", + "durationMs": 739, + "spanSet": { + "spans": [ + { + "spanID": "bfa708c276dadca1", + "startTimeUnixNano": "1795100493744533417", + "durationNanos": "120333", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "1c7bd95ce8f4cc87", + "startTimeUnixNano": "1795100493744692542", + "durationNanos": "161542", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "c96dfa07d20d7161", + "startTimeUnixNano": "1795100493744893417", + "durationNanos": "353792", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "bfa708c276dadca1", + "startTimeUnixNano": "1795100493744533417", + "durationNanos": "120333", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "1c7bd95ce8f4cc87", + "startTimeUnixNano": "1795100493744692542", + "durationNanos": "161542", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "c96dfa07d20d7161", + "startTimeUnixNano": "1795100493744893417", + "durationNanos": "353792", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 25 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "d2dd1ad79a101d8ef9ef393a56bae142", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100492853414875", + "durationMs": 644, + "spanSet": { + "spans": [ + { + "spanID": "c168ffa95885a3e9", + "startTimeUnixNano": "1795100493112498125", + "durationNanos": "163417", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "d7a51f97bb004bec", + "startTimeUnixNano": "1795100493112739292", + "durationNanos": "400958", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "b873270c966b09c8", + "startTimeUnixNano": "1795100493113219292", + "durationNanos": "249666", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "c168ffa95885a3e9", + "startTimeUnixNano": "1795100493112498125", + "durationNanos": "163417", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "d7a51f97bb004bec", + "startTimeUnixNano": "1795100493112739292", + "durationNanos": "400958", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "b873270c966b09c8", + "startTimeUnixNano": "1795100493113219292", + "durationNanos": "249666", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 25 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "2711cd248788a00577227ac44094fa66", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100492838905917", + "durationMs": 528, + "spanSet": { + "spans": [ + { + "spanID": "445c9e1d8ff4cb10", + "startTimeUnixNano": "1795100492978138958", + "durationNanos": "832584", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "cf04bd30a562e3a1", + "startTimeUnixNano": "1795100492979022917", + "durationNanos": "844583", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "58395985e025d7c8", + "startTimeUnixNano": "1795100492979923583", + "durationNanos": "288500", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "445c9e1d8ff4cb10", + "startTimeUnixNano": "1795100492978138958", + "durationNanos": "832584", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "cf04bd30a562e3a1", + "startTimeUnixNano": "1795100492979022917", + "durationNanos": "844583", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "58395985e025d7c8", + "startTimeUnixNano": "1795100492979923583", + "durationNanos": "288500", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 25 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "59ab46986444baabf270af63261c69fe", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100492684567792", + "durationMs": 426, + "spanSet": { + "spans": [ + { + "spanID": "d341fc37be8159c2", + "startTimeUnixNano": "1795100492722141125", + "durationNanos": "152958", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "3ef0cdebe8d774b9", + "startTimeUnixNano": "1795100492722332083", + "durationNanos": "346667", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "e9ab0ac65bd0d53b", + "startTimeUnixNano": "1795100492722747708", + "durationNanos": "269875", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "d341fc37be8159c2", + "startTimeUnixNano": "1795100492722141125", + "durationNanos": "152958", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "3ef0cdebe8d774b9", + "startTimeUnixNano": "1795100492722332083", + "durationNanos": "346667", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "e9ab0ac65bd0d53b", + "startTimeUnixNano": "1795100492722747708", + "durationNanos": "269875", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 25 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "d388b15ec1b1bdf0fb34e900d128f8f9", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100491993826833", + "durationMs": 605, + "spanSet": { + "spans": [ + { + "spanID": "c8f4d22650590384", + "startTimeUnixNano": "1795100492215401208", + "durationNanos": "219500", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "8841086e36d3b4b1", + "startTimeUnixNano": "1795100492215672750", + "durationNanos": "361583", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "90d506b456c4c2c3", + "startTimeUnixNano": "1795100492216097166", + "durationNanos": "238209", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "c8f4d22650590384", + "startTimeUnixNano": "1795100492215401208", + "durationNanos": "219500", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "8841086e36d3b4b1", + "startTimeUnixNano": "1795100492215672750", + "durationNanos": "361583", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "90d506b456c4c2c3", + "startTimeUnixNano": "1795100492216097166", + "durationNanos": "238209", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 25 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "783086f516c4e661bc869705126a2af8", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100491189085888", + "durationMs": 353, + "spanSet": { + "spans": [ + { + "spanID": "425a0e6656635775", + "startTimeUnixNano": "1795100491203815013", + "durationNanos": "158500", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "48fb8d7f90973481", + "startTimeUnixNano": "1795100491204031680", + "durationNanos": "838375", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "94901cc229da02fa", + "startTimeUnixNano": "1795100491204936722", + "durationNanos": "287750", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "425a0e6656635775", + "startTimeUnixNano": "1795100491203815013", + "durationNanos": "158500", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "48fb8d7f90973481", + "startTimeUnixNano": "1795100491204031680", + "durationNanos": "838375", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "94901cc229da02fa", + "startTimeUnixNano": "1795100491204936722", + "durationNanos": "287750", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 25 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + }, + { + "traceID": "9ca50ed8577cfb6dbc072818bad3a6e4", + "rootServiceName": "messenger-api", + "rootTraceName": "POST /api/v1/chats/{chat_id}/attachments", + "startTimeUnixNano": "1795100484893725219", + "durationMs": 722, + "spanSet": { + "spans": [ + { + "spanID": "be24bf246a6170f6", + "startTimeUnixNano": "1795100485210686136", + "durationNanos": "674625", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "3e8d0ff7bbfb2b55", + "startTimeUnixNano": "1795100485211487719", + "durationNanos": "2895667", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "d1ebfc174a704e5a", + "startTimeUnixNano": "1795100485214521469", + "durationNanos": "684375", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + }, + "spanSets": [ + { + "spans": [ + { + "spanID": "be24bf246a6170f6", + "startTimeUnixNano": "1795100485210686136", + "durationNanos": "674625", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "3e8d0ff7bbfb2b55", + "startTimeUnixNano": "1795100485211487719", + "durationNanos": "2895667", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + { + "spanID": "d1ebfc174a704e5a", + "startTimeUnixNano": "1795100485214521469", + "durationNanos": "684375", + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + } + ], + "matched": 20 + } + ], + "serviceStats": { + "messenger-api": { + "spanCount": 25 + }, + "messenger-publisher": { + "spanCount": 1 + }, + "messenger-worker": { + "spanCount": 20 + } + } + } + ], + "metrics": { + "inspectedBytes": "1257542", + "completedJobs": 3, + "totalJobs": 3 + } +} diff --git a/evidence/lab6-trace-retry.json b/evidence/lab6-trace-retry.json new file mode 100644 index 0000000..fd57b82 --- /dev/null +++ b/evidence/lab6-trace-retry.json @@ -0,0 +1,1701 @@ +{ + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "166e3f521d4f:1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "6.0.0" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "6.0.0" + }, + "spans": [ + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "4EwsdNCfoGY=", + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795100974757986751", + "endTimeUnixNano": "1795100974767815376", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/attachments" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "9u6vQxyMjao=", + "parentSpanId": "4EwsdNCfoGY=", + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795100974758449043", + "endTimeUnixNano": "1795100974759136959", + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "3M8AQ/xpH9c=", + "parentSpanId": "9u6vQxyMjao=", + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974758523918", + "endTimeUnixNano": "1795100974758656626", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "ANxAKR/V7OM=", + "parentSpanId": "9u6vQxyMjao=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974758720501", + "endTimeUnixNano": "1795100974759001834", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "uIn2sng7Kus=", + "parentSpanId": "4EwsdNCfoGY=", + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795100974759559168", + "endTimeUnixNano": "1795100974760490168", + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "OO7qlvUo5+I=", + "parentSpanId": "uIn2sng7Kus=", + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974759601668", + "endTimeUnixNano": "1795100974759705668", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "CPcEbjodphI=", + "parentSpanId": "uIn2sng7Kus=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974759763834", + "endTimeUnixNano": "1795100974760098501", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "pmirUgfy3bQ=", + "parentSpanId": "uIn2sng7Kus=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974760150001", + "endTimeUnixNano": "1795100974760367293", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "gZ1vISYZP/k=", + "parentSpanId": "4EwsdNCfoGY=", + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795100974762250418", + "endTimeUnixNano": "1795100974763237501", + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "e2uaMVnynHc=", + "parentSpanId": "gZ1vISYZP/k=", + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974762285876", + "endTimeUnixNano": "1795100974762373001", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "FjfJZlIOirI=", + "parentSpanId": "gZ1vISYZP/k=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974762427751", + "endTimeUnixNano": "1795100974762721709", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "TL4BIn72Nes=", + "parentSpanId": "gZ1vISYZP/k=", + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974762774084", + "endTimeUnixNano": "1795100974762937959", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "FeaV8dG3Ix0=", + "parentSpanId": "4EwsdNCfoGY=", + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795100974763277209", + "endTimeUnixNano": "1795100974767472793", + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "BEv15xAtjqw=", + "parentSpanId": "FeaV8dG3Ix0=", + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974763308626", + "endTimeUnixNano": "1795100974763380709", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "NR1M9MQaaVQ=", + "parentSpanId": "FeaV8dG3Ix0=", + "name": "postgresql.SET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974763428209", + "endTimeUnixNano": "1795100974763551793", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SET" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "hFv1zuFrDrM=", + "parentSpanId": "FeaV8dG3Ix0=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974763608751", + "endTimeUnixNano": "1795100974763710168", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "OOUUNzMj5rg=", + "parentSpanId": "FeaV8dG3Ix0=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974763752251", + "endTimeUnixNano": "1795100974763865168", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "PQxHPZ03y1M=", + "parentSpanId": "FeaV8dG3Ix0=", + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974763907043", + "endTimeUnixNano": "1795100974766021418", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "PutObject" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "tNikt9vTPsw=", + "parentSpanId": "FeaV8dG3Ix0=", + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974766102709", + "endTimeUnixNano": "1795100974766613126", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "iIJt8p5J3a4=", + "parentSpanId": "FeaV8dG3Ix0=", + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974766711709", + "endTimeUnixNano": "1795100974766868668", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "z6LkRMhwVKs=", + "parentSpanId": "FeaV8dG3Ix0=", + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974766911251", + "endTimeUnixNano": "1795100974767059793", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "aOxvFod3Fio=", + "parentSpanId": "FeaV8dG3Ix0=", + "name": "postgresql.DELETE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100974767116918", + "endTimeUnixNano": "1795100974767221459", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "DELETE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "0cd7cf668f46:1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "6.0.0" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-publisher" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "6.0.0" + }, + "spans": [ + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "hN+CphfJJO0=", + "parentSpanId": "FeaV8dG3Ix0=", + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1795100974810469209", + "endTimeUnixNano": "1795100974811784293", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachments:v1" + } + }, + { + "key": "messaging.operation.type", + "value": { + "stringValue": "send" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "4a4dd93b6332:8" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "6.0.0" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "6.0.0" + }, + "spans": [ + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "K7Ms03WYia4=", + "parentSpanId": "hN+CphfJJO0=", + "flags": 768, + "name": "attachments process", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1795100977539356419", + "endTimeUnixNano": "1795100977544230252", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachments:v1" + } + }, + { + "key": "messaging.operation.type", + "value": { + "stringValue": "process" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "1" + } + }, + { + "key": "job.outcome", + "value": { + "stringValue": "retry" + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + } + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "jzdx/YCVSc8=", + "parentSpanId": "K7Ms03WYia4=", + "flags": 256, + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795100977539487460", + "endTimeUnixNano": "1795100977541835127", + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "oXQyP11YsHc=", + "parentSpanId": "jzdx/YCVSc8=", + "flags": 256, + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100977539545960", + "endTimeUnixNano": "1795100977539708294", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "r4loWJe6MlQ=", + "parentSpanId": "jzdx/YCVSc8=", + "flags": 256, + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100977539772127", + "endTimeUnixNano": "1795100977540637044", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "jl1H1IWHKAY=", + "parentSpanId": "jzdx/YCVSc8=", + "flags": 256, + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100977540718919", + "endTimeUnixNano": "1795100977541074210", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "k5sbF/HL9IQ=", + "parentSpanId": "jzdx/YCVSc8=", + "flags": 256, + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100977541136335", + "endTimeUnixNano": "1795100977541291460", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "BBqg1veWYDU=", + "parentSpanId": "jzdx/YCVSc8=", + "flags": 256, + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100977541336252", + "endTimeUnixNano": "1795100977541594627", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "+f8oHT2/rt0=", + "parentSpanId": "K7Ms03WYia4=", + "flags": 256, + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795100977542004127", + "endTimeUnixNano": "1795100977543460419", + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "66rBTSPLXEo=", + "parentSpanId": "+f8oHT2/rt0=", + "flags": 256, + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100977542041919", + "endTimeUnixNano": "1795100977542107960", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "SJApU1YYzc8=", + "parentSpanId": "+f8oHT2/rt0=", + "flags": 256, + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100977542155377", + "endTimeUnixNano": "1795100977542370002", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "PHRVuwtkChk=", + "parentSpanId": "+f8oHT2/rt0=", + "flags": 256, + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100977542432210", + "endTimeUnixNano": "1795100977542568710", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "1pOg3orRveM=", + "parentSpanId": "+f8oHT2/rt0=", + "flags": 256, + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100977542608127", + "endTimeUnixNano": "1795100977542707377", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "vjgHaUYeke8=", + "parentSpanId": "+f8oHT2/rt0=", + "flags": 256, + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100977542744044", + "endTimeUnixNano": "1795100977542878335", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "OYD6PTfHhmE=", + "parentSpanId": "+f8oHT2/rt0=", + "flags": 256, + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100977542914794", + "endTimeUnixNano": "1795100977542999252", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "6/w+wBW/JCQ=", + "parentSpanId": "+f8oHT2/rt0=", + "flags": 256, + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100977543042752", + "endTimeUnixNano": "1795100977543150502", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "0cd7cf668f46:1" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-publisher" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "6.0.0" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "6.0.0" + }, + "spans": [ + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "1OusVVkLWj8=", + "parentSpanId": "+f8oHT2/rt0=", + "flags": 768, + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1795100978861827586", + "endTimeUnixNano": "1795100978865714586", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachments:v1" + } + }, + { + "key": "messaging.operation.type", + "value": { + "stringValue": "send" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "2" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "4a4dd93b6332:9" + } + }, + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "6.0.0" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "6.0.0" + }, + "spans": [ + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "JrAZPBhfh2E=", + "parentSpanId": "1OusVVkLWj8=", + "flags": 768, + "name": "attachments process", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1795100978867277294", + "endTimeUnixNano": "1795100979222472628", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachments:v1" + } + }, + { + "key": "messaging.operation.type", + "value": { + "stringValue": "process" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "2" + } + }, + { + "key": "job.outcome", + "value": { + "stringValue": "ready" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "5j/KAzpZJEU=", + "parentSpanId": "JrAZPBhfh2E=", + "flags": 256, + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795100978867880294", + "endTimeUnixNano": "1795100978876843503", + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "Fx0EzUJQ8Uw=", + "parentSpanId": "5j/KAzpZJEU=", + "flags": 256, + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100978868277419", + "endTimeUnixNano": "1795100978868830044", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "+6ROxske6G4=", + "parentSpanId": "5j/KAzpZJEU=", + "flags": 256, + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100978869151044", + "endTimeUnixNano": "1795100978872780628", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "r4kkVEgzIlE=", + "parentSpanId": "5j/KAzpZJEU=", + "flags": 256, + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100978873072669", + "endTimeUnixNano": "1795100978874329919", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "wrhORk9JQHs=", + "parentSpanId": "5j/KAzpZJEU=", + "flags": 256, + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100978874581419", + "endTimeUnixNano": "1795100978875173586", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "/dPedGwQAYo=", + "parentSpanId": "5j/KAzpZJEU=", + "flags": 256, + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100978875369711", + "endTimeUnixNano": "1795100978876053128", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "AMW42Us7Z3E=", + "parentSpanId": "JrAZPBhfh2E=", + "flags": 256, + "name": "s3.get_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100978877633044", + "endTimeUnixNano": "1795100978881334086", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "GetObject" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "SeM+hw5erk4=", + "parentSpanId": "JrAZPBhfh2E=", + "flags": 256, + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795100979215230170", + "endTimeUnixNano": "1795100979221315753", + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "hjSxq3xjpj8=", + "parentSpanId": "SeM+hw5erk4=", + "flags": 256, + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100979215304961", + "endTimeUnixNano": "1795100979215440128", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "wu3cImnCde4=", + "parentSpanId": "SeM+hw5erk4=", + "flags": 256, + "name": "postgresql.SET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100979215506295", + "endTimeUnixNano": "1795100979215654545", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SET" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "jCsjj1pCHVk=", + "parentSpanId": "SeM+hw5erk4=", + "flags": 256, + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100979215721628", + "endTimeUnixNano": "1795100979216060586", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "lXHzazYKs+M=", + "parentSpanId": "SeM+hw5erk4=", + "flags": 256, + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100979216149795", + "endTimeUnixNano": "1795100979216319420", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "rjdv0gFElpk=", + "parentSpanId": "SeM+hw5erk4=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100979216409670", + "endTimeUnixNano": "1795100979218276545", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "PutObject" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "JgGJZlKVJJ8=", + "parentSpanId": "SeM+hw5erk4=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100979218344253", + "endTimeUnixNano": "1795100979219080003", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "PutObject" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "oZwk9cIsBfw=", + "parentSpanId": "SeM+hw5erk4=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100979219140878", + "endTimeUnixNano": "1795100979219690211", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "PutObject" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "naT8OO0WfrY=", + "parentSpanId": "SeM+hw5erk4=", + "flags": 256, + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100979219746253", + "endTimeUnixNano": "1795100979220303545", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "PutObject" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "LWO9mqDCcBQ=", + "parentSpanId": "SeM+hw5erk4=", + "flags": 256, + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100979220400503", + "endTimeUnixNano": "1795100979220675336", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "6oIt+dqEpb8=", + "parentSpanId": "SeM+hw5erk4=", + "flags": 256, + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100979220722336", + "endTimeUnixNano": "1795100979220845503", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "k+EXf5Vo9bv8C2lnoTJKhA==", + "spanId": "f+hc6K0fX7E=", + "parentSpanId": "SeM+hw5erk4=", + "flags": 256, + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795100979220889003", + "endTimeUnixNano": "1795100979221024545", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + } + ] + } + ] + } + ] +} diff --git a/evidence/lab6-worker-incident-logs.json b/evidence/lab6-worker-incident-logs.json new file mode 100644 index 0000000..12dcdea --- /dev/null +++ b/evidence/lab6-worker-incident-logs.json @@ -0,0 +1,248 @@ +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "attachment_id": "d0c6d064-5a57-47eb-a5b1-c8d2f7a9870f", + "deployment_environment_name": "lab", + "detected_level": "INFO", + "event": "attachment.accepted", + "flags": "3", + "job_id": "03320707-a810-4a0e-a72d-b10ef7734c32", + "scope_name": "messenger", + "scope_version": "6.0.0", + "service_instance_id": "33daefd247cf:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "65e8265a35af1c1d", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1", + "trace_id": "2b1592ca18aac08bbd14b3f1825f0816" + }, + "values": [ + [ + "1795102383403137083", + "{\"timestamp\": \"2026-11-19T15:33:03.403092+00:00\", \"severity\": \"INFO\", \"service.name\": \"messenger-api\", \"event\": \"attachment.accepted\", \"trace_id\": \"2b1592ca18aac08bbd14b3f1825f0816\", \"span_id\": \"65e8265a35af1c1d\", \"job_id\": \"03320707-a810-4a0e-a72d-b10ef7734c32\", \"attachment_id\": \"d0c6d064-5a57-47eb-a5b1-c8d2f7a9870f\"}" + ] + ] + }, + { + "stream": { + "attempt": "1", + "deployment_environment_name": "lab", + "detected_level": "INFO", + "event": "job.finished", + "flags": "3", + "job_id": "03320707-a810-4a0e-a72d-b10ef7734c32", + "outcome": "ready", + "scope_name": "messenger", + "scope_version": "6.0.0", + "service_instance_id": "168b98028f54:8", + "service_name": "messenger-worker", + "service_version": "6.0.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "ba2d3f8e53a54fd2", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1", + "trace_id": "2b1592ca18aac08bbd14b3f1825f0816" + }, + "values": [ + [ + "1795102477984404627", + "{\"timestamp\": \"2026-11-19T15:34:37.984358+00:00\", \"severity\": \"INFO\", \"service.name\": \"messenger-worker\", \"event\": \"job.finished\", \"trace_id\": \"2b1592ca18aac08bbd14b3f1825f0816\", \"span_id\": \"ba2d3f8e53a54fd2\", \"job_id\": \"03320707-a810-4a0e-a72d-b10ef7734c32\", \"attempt\": 1, \"outcome\": \"ready\"}" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "INFO", + "duration_seconds": "0.024863", + "event": "http.request", + "flags": "3", + "method": "POST", + "request_id": "bad9f74b-1237-475e-b7a9-13e04b9457fb", + "route": "/api/v1/chats/{chat_id}/attachments", + "scope_name": "messenger", + "scope_version": "6.0.0", + "service_instance_id": "33daefd247cf:1", + "service_name": "messenger-api", + "service_version": "6.0.0", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "65e8265a35af1c1d", + "status": "202", + "telemetry_sdk_language": "python", + "telemetry_sdk_name": "opentelemetry", + "telemetry_sdk_version": "1.45.1", + "trace_id": "2b1592ca18aac08bbd14b3f1825f0816" + }, + "values": [ + [ + "1795102383403584208", + "{\"timestamp\": \"2026-11-19T15:33:03.403545+00:00\", \"severity\": \"INFO\", \"service.name\": \"messenger-api\", \"event\": \"http.request\", \"trace_id\": \"2b1592ca18aac08bbd14b3f1825f0816\", \"span_id\": \"65e8265a35af1c1d\", \"method\": \"POST\", \"route\": \"/api/v1/chats/{chat_id}/attachments\", \"status\": 202, \"duration_seconds\": 0.024863, \"request_id\": \"bad9f74b-1237-475e-b7a9-13e04b9457fb\"}" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 99017520, + "linesProcessedPerSecond": 135509, + "totalBytesProcessed": 2015283, + "totalLinesProcessed": 2758, + "execTime": 0.020353, + "queueTime": 0.000121, + "subqueries": 0, + "totalEntriesReturned": 3, + "splits": 0, + "shards": 1, + "totalPostFilterLines": 3, + "totalStructuredMetadataBytesProcessed": 1011858 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 1, + "totalChunksMatched": 2, + "totalBatches": 2, + "totalLinesSent": 3, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 39416, + "headChunkLines": 85, + "decompressedBytes": 1975867, + "decompressedLines": 2673, + "compressedBytes": 241323, + "totalDuplicates": 0, + "postFilterLines": 3, + "headChunkStructuredMetadataBytes": 12008, + "decompressedStructuredMetadataBytes": 999850 + }, + "chunkRefsFetchTime": 293083, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false + } + } + } +} diff --git a/evidence/lab6-worker-incident-trace.json b/evidence/lab6-worker-incident-trace.json new file mode 100644 index 0000000..c751c31 --- /dev/null +++ b/evidence/lab6-worker-incident-trace.json @@ -0,0 +1,1161 @@ +{ + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "33daefd247cf:1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "6.0.0" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "6.0.0" + }, + "spans": [ + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "ZegmWjWvHB0=", + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1795102383379066458", + "endTimeUnixNano": "1795102383403622667", + "attributes": [ + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/attachments" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "KYI80CM/0t8=", + "parentSpanId": "ZegmWjWvHB0=", + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795102383380404917", + "endTimeUnixNano": "1795102383383171250", + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "JvcOiIDqzCA=", + "parentSpanId": "KYI80CM/0t8=", + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383380706208", + "endTimeUnixNano": "1795102383381860500", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "FjoFl91PeS8=", + "parentSpanId": "KYI80CM/0t8=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383382207500", + "endTimeUnixNano": "1795102383382880292", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "nPUgW6Em8oE=", + "parentSpanId": "ZegmWjWvHB0=", + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795102383385015250", + "endTimeUnixNano": "1795102383387045208", + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "V+slhANcUSY=", + "parentSpanId": "nPUgW6Em8oE=", + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383385157083", + "endTimeUnixNano": "1795102383385395167", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "XKF5DKZPuzg=", + "parentSpanId": "nPUgW6Em8oE=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383385773417", + "endTimeUnixNano": "1795102383386225375", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "o6eCVH6IzCc=", + "parentSpanId": "nPUgW6Em8oE=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383386463958", + "endTimeUnixNano": "1795102383386782417", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "t3AcVqkOhNg=", + "parentSpanId": "ZegmWjWvHB0=", + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795102383396736833", + "endTimeUnixNano": "1795102383398664292", + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "vfIW1QrPFOQ=", + "parentSpanId": "t3AcVqkOhNg=", + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383396873167", + "endTimeUnixNano": "1795102383397127250", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "gW0H12II3mU=", + "parentSpanId": "t3AcVqkOhNg=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383397256958", + "endTimeUnixNano": "1795102383397836583", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "PUv6H3ZwZao=", + "parentSpanId": "t3AcVqkOhNg=", + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383397942458", + "endTimeUnixNano": "1795102383398163208", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "WjkIIL163IY=", + "parentSpanId": "ZegmWjWvHB0=", + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795102383398746875", + "endTimeUnixNano": "1795102383403068542", + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "T7TGA4kemSM=", + "parentSpanId": "WjkIIL163IY=", + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383398809208", + "endTimeUnixNano": "1795102383398920000", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "bc5sgF8fyUw=", + "parentSpanId": "WjkIIL163IY=", + "name": "postgresql.SET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383398999167", + "endTimeUnixNano": "1795102383399135750", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SET" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "e1DG/FvLysY=", + "parentSpanId": "WjkIIL163IY=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383399206250", + "endTimeUnixNano": "1795102383399306875", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "9AS4IcW3BGg=", + "parentSpanId": "WjkIIL163IY=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383399377875", + "endTimeUnixNano": "1795102383399486292", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "dQhQFcRn7x8=", + "parentSpanId": "WjkIIL163IY=", + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383399551375", + "endTimeUnixNano": "1795102383401901833", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "PutObject" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "mE0npEZdYJY=", + "parentSpanId": "WjkIIL163IY=", + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383401990083", + "endTimeUnixNano": "1795102383402241917", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "EECSxZA5VDw=", + "parentSpanId": "WjkIIL163IY=", + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383402346917", + "endTimeUnixNano": "1795102383402492375", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "sLJDjquVH10=", + "parentSpanId": "WjkIIL163IY=", + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383402552250", + "endTimeUnixNano": "1795102383402659208", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "Ko3anoGk5nY=", + "parentSpanId": "WjkIIL163IY=", + "name": "postgresql.DELETE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102383402723500", + "endTimeUnixNano": "1795102383402802250", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "DELETE" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "02738dc58bbb:1" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "6.0.0" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-publisher" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "6.0.0" + }, + "spans": [ + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "Nixq2Xr8Tlc=", + "parentSpanId": "WjkIIL163IY=", + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1795102383484171458", + "endTimeUnixNano": "1795102383485732875", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachments:v1" + } + }, + { + "key": "messaging.operation.type", + "value": { + "stringValue": "send" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "1" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "168b98028f54:8" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "6.0.0" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.1" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger", + "version": "6.0.0" + }, + "spans": [ + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "ui0/jlOlT9I=", + "parentSpanId": "Nixq2Xr8Tlc=", + "name": "attachments process", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1795102477649825377", + "endTimeUnixNano": "1795102477984442919", + "attributes": [ + { + "key": "messaging.system", + "value": { + "stringValue": "redis" + } + }, + { + "key": "messaging.destination.name", + "value": { + "stringValue": "attachments:v1" + } + }, + { + "key": "messaging.operation.type", + "value": { + "stringValue": "process" + } + }, + { + "key": "job.attempt", + "value": { + "intValue": "1" + } + }, + { + "key": "job.outcome", + "value": { + "stringValue": "ready" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "MzjNX9XWEVk=", + "parentSpanId": "ui0/jlOlT9I=", + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795102477649876960", + "endTimeUnixNano": "1795102477650653460", + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "kogJs8FZxtI=", + "parentSpanId": "MzjNX9XWEVk=", + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477649912502", + "endTimeUnixNano": "1795102477649969335", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "tmmkjypTVFw=", + "parentSpanId": "MzjNX9XWEVk=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477650010169", + "endTimeUnixNano": "1795102477650115044", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "kbWfAhBt91k=", + "parentSpanId": "MzjNX9XWEVk=", + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477650168585", + "endTimeUnixNano": "1795102477650295044", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "9s3fcHgPAg4=", + "parentSpanId": "MzjNX9XWEVk=", + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477650332460", + "endTimeUnixNano": "1795102477650417085", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "uDJRO6S3uD8=", + "parentSpanId": "MzjNX9XWEVk=", + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477650455835", + "endTimeUnixNano": "1795102477650514294", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "iu/rk9hR0XE=", + "parentSpanId": "ui0/jlOlT9I=", + "name": "s3.get_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477650808752", + "endTimeUnixNano": "1795102477651362669", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "GetObject" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "ovA/yRiT1m0=", + "parentSpanId": "ui0/jlOlT9I=", + "name": "postgresql.transaction", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1795102477978549836", + "endTimeUnixNano": "1795102477983504919", + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "ltyemt9/MjU=", + "parentSpanId": "ovA/yRiT1m0=", + "name": "postgresql.QUERY", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477978606961", + "endTimeUnixNano": "1795102477978766211", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "QUERY" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "6Yrd5cTe3r4=", + "parentSpanId": "ovA/yRiT1m0=", + "name": "postgresql.SET", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477978815961", + "endTimeUnixNano": "1795102477978897044", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SET" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "3KTmurrMrFE=", + "parentSpanId": "ovA/yRiT1m0=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477978934752", + "endTimeUnixNano": "1795102477979058169", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "iTkWfWrbP2c=", + "parentSpanId": "ovA/yRiT1m0=", + "name": "postgresql.SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477979112502", + "endTimeUnixNano": "1795102477979194127", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "e4nrCRbOvZ4=", + "parentSpanId": "ovA/yRiT1m0=", + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477979240586", + "endTimeUnixNano": "1795102477980624586", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "PutObject" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "5ONHP3vrUNs=", + "parentSpanId": "ovA/yRiT1m0=", + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477980688836", + "endTimeUnixNano": "1795102477981292836", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "PutObject" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "qLS4ue/m1nU=", + "parentSpanId": "ovA/yRiT1m0=", + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477981356461", + "endTimeUnixNano": "1795102477981975419", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "PutObject" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "KI4viNQWoBY=", + "parentSpanId": "ovA/yRiT1m0=", + "name": "s3.put_object", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477982028961", + "endTimeUnixNano": "1795102477982626002", + "attributes": [ + { + "key": "rpc.system", + "value": { + "stringValue": "s3" + } + }, + { + "key": "rpc.method", + "value": { + "stringValue": "PutObject" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "OES8WcwL4i0=", + "parentSpanId": "ovA/yRiT1m0=", + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477982676836", + "endTimeUnixNano": "1795102477982873252", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "Dn1yr0xDGio=", + "parentSpanId": "ovA/yRiT1m0=", + "name": "postgresql.UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477982920502", + "endTimeUnixNano": "1795102477983036961", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "KxWSyhiqwIu9FLPxgl8IFg==", + "spanId": "5cmyTUU8aJs=", + "parentSpanId": "ovA/yRiT1m0=", + "name": "postgresql.INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1795102477983081919", + "endTimeUnixNano": "1795102477983182252", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + } + ] + } + ] + } + ] +} diff --git a/evidence/postgres-pause.json b/evidence/postgres-pause.json new file mode 100644 index 0000000..7338271 --- /dev/null +++ b/evidence/postgres-pause.json @@ -0,0 +1,76 @@ +{ + "utc": "2026-11-06T07:27:40.514878+00:00", + "project": "lab4-pg-pause-30f1d978ce", + "command": ".venv/bin/python student-tests/postgres_pause.py", + "passed": true, + "sql_without_server_timeout": { + "elapsed_seconds": 2.001, + "connection_closed": true + }, + "direct_ready": { + "status": 503, + "elapsed_seconds": 2.012 + }, + "direct_live": { + "status": 200, + "elapsed_seconds": 0.006 + }, + "repeated_ready": [ + { + "status": 503, + "elapsed_seconds": 2.11 + }, + { + "status": 503, + "elapsed_seconds": 2.011 + }, + { + "status": 503, + "elapsed_seconds": 2.014 + }, + { + "status": 503, + "elapsed_seconds": 2.097 + }, + { + "status": 503, + "elapsed_seconds": 2.194 + }, + { + "status": 503, + "elapsed_seconds": 2.013 + }, + { + "status": 503, + "elapsed_seconds": 2.299 + }, + { + "status": 503, + "elapsed_seconds": 2.203 + }, + { + "status": 503, + "elapsed_seconds": 2.013 + }, + { + "status": 503, + "elapsed_seconds": 2.094 + } + ], + "ready_after_12_seconds": { + "status": 503, + "elapsed_seconds": 0.009 + }, + "live_after_repeated_requests": { + "status": 200, + "elapsed_seconds": 0.006 + }, + "paused_seconds": 12.565, + "recovered_ready": { + "status": 200, + "elapsed_seconds": 0.009 + }, + "recovery_seconds": 2.874, + "api_not_restarted": true, + "existing_session_and_message_preserved": true +} diff --git a/evidence/query-plans.json b/evidence/query-plans.json new file mode 100644 index 0000000..25f478d --- /dev/null +++ b/evidence/query-plans.json @@ -0,0 +1,2621 @@ +[ + { + "variant": "chat_only", + "chat_size": 70000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 5732.59, + "Total Cost": 5738.45, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 7.421, + "Actual Total Time": 8.522, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 5732.59, + "Total Cost": 10128.12, + "Plan Rows": 38222, + "Plan Width": 160, + "Actual Startup Time": 7.421, + "Actual Total Time": 8.508, + "Actual Rows": 51, + "Actual Loops": 1, + "Workers Planned": 1, + "Workers Launched": 1, + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 4732.58, + "Total Cost": 4828.13, + "Plan Rows": 38222, + "Plan Width": 160, + "Actual Startup Time": 6.621, + "Actual Total Time": 6.624, + "Actual Rows": 51, + "Actual Loops": 2, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Workers": [ + { + "Worker Number": 0, + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory" + } + ], + "Plans": [ + { + "Node Type": "Seq Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.0, + "Total Cost": 3457.41, + "Plan Rows": 38222, + "Plan Width": 160, + "Actual Startup Time": 0.215, + "Actual Total Time": 4.164, + "Actual Rows": 31500, + "Actual Loops": 2, + "Filter": "((chat_id = 'fb372790-4b73-5a45-c97c-7e4287fa05be'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.007+00'::timestamp with time zone, '9cc0c1db-51d1-5bf2-e5a4-83cb93303974'::uuid)))", + "Rows Removed by Filter": 18500, + "Shared Hit Blocks": 2428, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.031, + "Triggers": [], + "Execution Time": 8.532 + } + }, + { + "variant": "chat_only", + "chat_size": 70000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 5354.06, + "Total Cost": 5359.93, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 5.792, + "Actual Total Time": 7.033, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 5354.06, + "Total Cost": 8444.8, + "Plan Rows": 26876, + "Plan Width": 160, + "Actual Startup Time": 5.792, + "Actual Total Time": 7.028, + "Actual Rows": 51, + "Actual Loops": 1, + "Workers Planned": 1, + "Workers Launched": 1, + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 4354.05, + "Total Cost": 4421.24, + "Plan Rows": 26876, + "Plan Width": 160, + "Actual Startup Time": 4.884, + "Actual Total Time": 4.886, + "Actual Rows": 40, + "Actual Loops": 2, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Workers": [ + { + "Worker Number": 0, + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory" + } + ], + "Plans": [ + { + "Node Type": "Seq Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.0, + "Total Cost": 3457.41, + "Plan Rows": 26876, + "Plan Width": 160, + "Actual Startup Time": 1.017, + "Actual Total Time": 3.623, + "Actual Rows": 17500, + "Actual Loops": 2, + "Filter": "((chat_id = 'fb372790-4b73-5a45-c97c-7e4287fa05be'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.035+00'::timestamp with time zone, '169ee7a7-ca0d-aa05-bd8c-a1eb275708e7'::uuid)))", + "Rows Removed by Filter": 32500, + "Shared Hit Blocks": 2428, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.038, + "Triggers": [], + "Execution Time": 7.042 + } + }, + { + "variant": "chat_only", + "chat_size": 70000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 4967.83, + "Total Cost": 4973.69, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 25.942, + "Actual Total Time": 27.248, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 4967.83, + "Total Cost": 6727.21, + "Plan Rows": 15299, + "Plan Width": 160, + "Actual Startup Time": 25.942, + "Actual Total Time": 27.243, + "Actual Rows": 51, + "Actual Loops": 1, + "Workers Planned": 1, + "Workers Launched": 1, + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3967.82, + "Total Cost": 4006.07, + "Plan Rows": 15299, + "Plan Width": 160, + "Actual Startup Time": 14.755, + "Actual Total Time": 14.763, + "Actual Rows": 26, + "Actual Loops": 2, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 2443, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Workers": [ + { + "Worker Number": 0, + "Sort Method": "quicksort", + "Sort Space Used": 25, + "Sort Space Type": "Memory" + } + ], + "Plans": [ + { + "Node Type": "Seq Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.0, + "Total Cost": 3457.41, + "Plan Rows": 15299, + "Plan Width": 160, + "Actual Startup Time": 13.398, + "Actual Total Time": 14.487, + "Actual Rows": 3500, + "Actual Loops": 2, + "Filter": "((chat_id = 'fb372790-4b73-5a45-c97c-7e4287fa05be'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.063+00'::timestamp with time zone, '987963b5-9cc6-d246-e263-52373e7f4b96'::uuid)))", + "Rows Removed by Filter": 46500, + "Shared Hit Blocks": 2428, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.047, + "Triggers": [], + "Execution Time": 27.257 + } + }, + { + "variant": "chat_only", + "chat_size": 20000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3191.11, + "Total Cost": 3191.24, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 3.535, + "Actual Total Time": 3.542, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3191.11, + "Total Cost": 3205.3, + "Plan Rows": 5677, + "Plan Width": 160, + "Actual Startup Time": 3.534, + "Actual Total Time": 3.538, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 220.91, + "Total Cost": 3001.71, + "Plan Rows": 5677, + "Plan Width": 160, + "Actual Startup Time": 0.442, + "Actual Total Time": 2.23, + "Actual Rows": 18000, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.072+00'::timestamp with time zone, '97b900c6-4a0c-f323-4d98-d97c50488136'::uuid))", + "Rows Removed by Filter": 2000, + "Exact Heap Blocks": 489, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 219.49, + "Plan Rows": 20160, + "Plan Width": 0, + "Actual Startup Time": 0.29, + "Actual Total Time": 0.291, + "Actual Rows": 20000, + "Actual Loops": 1, + "Index Cond": "(chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid)", + "Shared Hit Blocks": 17, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.043, + "Triggers": [], + "Execution Time": 3.549 + } + }, + { + "variant": "chat_only", + "chat_size": 20000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3136.93, + "Total Cost": 3137.05, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 2.648, + "Actual Total Time": 2.654, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3136.93, + "Total Cost": 3147.09, + "Plan Rows": 4065, + "Plan Width": 160, + "Actual Startup Time": 2.647, + "Actual Total Time": 2.65, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 220.51, + "Total Cost": 3001.31, + "Plan Rows": 4065, + "Plan Width": 160, + "Actual Startup Time": 0.94, + "Actual Total Time": 1.93, + "Actual Rows": 10000, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.08+00'::timestamp with time zone, '9042bcb2-df86-667b-9e54-e6e6c0c7923a'::uuid))", + "Rows Removed by Filter": 10000, + "Exact Heap Blocks": 489, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 219.49, + "Plan Rows": 20160, + "Plan Width": 0, + "Actual Startup Time": 0.276, + "Actual Total Time": 0.276, + "Actual Rows": 20000, + "Actual Loops": 1, + "Index Cond": "(chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid)", + "Shared Hit Blocks": 17, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.03, + "Triggers": [], + "Execution Time": 2.66 + } + }, + { + "variant": "chat_only", + "chat_size": 20000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3081.5, + "Total Cost": 3081.63, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 1.813, + "Actual Total Time": 1.819, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 3081.5, + "Total Cost": 3087.54, + "Plan Rows": 2416, + "Plan Width": 160, + "Actual Startup Time": 1.813, + "Actual Total Time": 1.816, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 220.1, + "Total Cost": 3000.9, + "Plan Rows": 2416, + "Plan Width": 160, + "Actual Startup Time": 1.459, + "Actual Total Time": 1.667, + "Actual Rows": 2000, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.088+00'::timestamp with time zone, '81028714-cb0a-8e0f-6d74-4c379351efc2'::uuid))", + "Rows Removed by Filter": 18000, + "Exact Heap Blocks": 489, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 506, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 219.49, + "Plan Rows": 20160, + "Plan Width": 0, + "Actual Startup Time": 0.277, + "Actual Total Time": 0.277, + "Actual Rows": 20000, + "Actual Loops": 1, + "Index Cond": "(chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid)", + "Shared Hit Blocks": 17, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.027, + "Triggers": [], + "Execution Time": 1.827 + } + }, + { + "variant": "chat_only", + "chat_size": 9000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 2710.71, + "Total Cost": 2710.84, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 1.58, + "Actual Total Time": 1.586, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 2710.71, + "Total Cost": 2712.76, + "Plan Rows": 818, + "Plan Width": 160, + "Actual Startup Time": 1.58, + "Actual Total Time": 1.582, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 99.37, + "Total Cost": 2683.42, + "Plan Rows": 818, + "Plan Width": 160, + "Actual Startup Time": 0.192, + "Actual Total Time": 1.001, + "Actual Rows": 8100, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.0909+00'::timestamp with time zone, 'b4d17acc-f297-d5d5-746e-2ef777eaa364'::uuid))", + "Rows Removed by Filter": 900, + "Exact Heap Blocks": 221, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 99.17, + "Plan Rows": 8917, + "Plan Width": 0, + "Actual Startup Time": 0.121, + "Actual Total Time": 0.121, + "Actual Rows": 9000, + "Actual Loops": 1, + "Index Cond": "(chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid)", + "Shared Hit Blocks": 9, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.021, + "Triggers": [], + "Execution Time": 1.592 + } + }, + { + "variant": "chat_only", + "chat_size": 9000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 2699.82, + "Total Cost": 2699.95, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 1.21, + "Actual Total Time": 1.216, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 2699.82, + "Total Cost": 2701.06, + "Plan Rows": 494, + "Plan Width": 160, + "Actual Startup Time": 1.209, + "Actual Total Time": 1.212, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 99.29, + "Total Cost": 2683.34, + "Plan Rows": 494, + "Plan Width": 160, + "Actual Startup Time": 0.437, + "Actual Total Time": 0.889, + "Actual Rows": 4500, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.0945+00'::timestamp with time zone, '64cc6094-35b0-52bc-2c33-655d810cbabc'::uuid))", + "Rows Removed by Filter": 4500, + "Exact Heap Blocks": 221, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 99.17, + "Plan Rows": 8917, + "Plan Width": 0, + "Actual Startup Time": 0.121, + "Actual Total Time": 0.121, + "Actual Rows": 9000, + "Actual Loops": 1, + "Index Cond": "(chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid)", + "Shared Hit Blocks": 9, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.02, + "Triggers": [], + "Execution Time": 1.222 + } + }, + { + "variant": "chat_only", + "chat_size": 9000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 2689.0, + "Total Cost": 2689.13, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.834, + "Actual Total Time": 0.84, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 2689.0, + "Total Cost": 2689.43, + "Plan Rows": 172, + "Plan Width": 160, + "Actual Startup Time": 0.834, + "Actual Total Time": 0.836, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 99.21, + "Total Cost": 2683.26, + "Plan Rows": 172, + "Plan Width": 160, + "Actual Startup Time": 0.661, + "Actual Total Time": 0.766, + "Actual Rows": 900, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.0981+00'::timestamp with time zone, '8de6f8f9-25ec-97e3-e3b7-59eac1d2fd4c'::uuid))", + "Rows Removed by Filter": 8100, + "Exact Heap Blocks": 221, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 230, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 99.17, + "Plan Rows": 8917, + "Plan Width": 0, + "Actual Startup Time": 0.12, + "Actual Total Time": 0.12, + "Actual Rows": 9000, + "Actual Loops": 1, + "Index Cond": "(chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid)", + "Shared Hit Blocks": 9, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.035, + "Triggers": [], + "Execution Time": 0.846 + } + }, + { + "variant": "chat_only", + "chat_size": 1000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1993.42, + "Total Cost": 1993.45, + "Plan Rows": 11, + "Plan Width": 160, + "Actual Startup Time": 0.192, + "Actual Total Time": 0.198, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1993.42, + "Total Cost": 1993.45, + "Plan Rows": 11, + "Plan Width": 160, + "Actual Startup Time": 0.192, + "Actual Total Time": 0.194, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 12.62, + "Total Cost": 1993.23, + "Plan Rows": 11, + "Plan Width": 160, + "Actual Startup Time": 0.024, + "Actual Total Time": 0.125, + "Actual Rows": 900, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.0991+00'::timestamp with time zone, 'a1580a3c-2826-e768-e4ea-f6c5a0e7171b'::uuid))", + "Rows Removed by Filter": 100, + "Exact Heap Blocks": 25, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 12.62, + "Plan Rows": 1110, + "Plan Width": 0, + "Actual Startup Time": 0.015, + "Actual Total Time": 0.015, + "Actual Rows": 1000, + "Actual Loops": 1, + "Index Cond": "(chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid)", + "Shared Hit Blocks": 2, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.02, + "Triggers": [], + "Execution Time": 0.204 + } + }, + { + "variant": "chat_only", + "chat_size": 1000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1993.31, + "Total Cost": 1993.32, + "Plan Rows": 6, + "Plan Width": 160, + "Actual Startup Time": 0.139, + "Actual Total Time": 0.145, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1993.31, + "Total Cost": 1993.32, + "Plan Rows": 6, + "Plan Width": 160, + "Actual Startup Time": 0.139, + "Actual Total Time": 0.141, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 38, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 12.62, + "Total Cost": 1993.23, + "Plan Rows": 6, + "Plan Width": 160, + "Actual Startup Time": 0.051, + "Actual Total Time": 0.1, + "Actual Rows": 500, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.0995+00'::timestamp with time zone, '86263544-3ca9-88ac-6a18-01fe3ed3d579'::uuid))", + "Rows Removed by Filter": 500, + "Exact Heap Blocks": 25, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 12.62, + "Plan Rows": 1110, + "Plan Width": 0, + "Actual Startup Time": 0.015, + "Actual Total Time": 0.015, + "Actual Rows": 1000, + "Actual Loops": 1, + "Index Cond": "(chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid)", + "Shared Hit Blocks": 2, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.019, + "Triggers": [], + "Execution Time": 0.151 + } + }, + { + "variant": "chat_only", + "chat_size": 1000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1993.24, + "Total Cost": 1993.24, + "Plan Rows": 1, + "Plan Width": 160, + "Actual Startup Time": 0.095, + "Actual Total Time": 0.101, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1993.24, + "Total Cost": 1993.24, + "Plan Rows": 1, + "Plan Width": 160, + "Actual Startup Time": 0.095, + "Actual Total Time": 0.097, + "Actual Rows": 51, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "quicksort", + "Sort Space Used": 50, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 12.62, + "Total Cost": 1993.23, + "Plan Rows": 1, + "Plan Width": 160, + "Actual Startup Time": 0.077, + "Actual Total Time": 0.086, + "Actual Rows": 100, + "Actual Loops": 1, + "Recheck Cond": "(chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2026-01-01 00:00:00.0999+00'::timestamp with time zone, 'dc2ec078-4b31-5bfc-d9ac-d4e6671f761b'::uuid))", + "Rows Removed by Filter": 900, + "Exact Heap Blocks": 25, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 27, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "messages_chat_idx", + "Startup Cost": 0.0, + "Total Cost": 12.62, + "Plan Rows": 1110, + "Plan Width": 0, + "Actual Startup Time": 0.015, + "Actual Total Time": 0.015, + "Actual Rows": 1000, + "Actual Loops": 1, + "Index Cond": "(chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid)", + "Shared Hit Blocks": 2, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.02, + "Triggers": [], + "Execution Time": 0.106 + } + }, + { + "variant": "history", + "chat_size": 70000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 10.63, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.014, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 13039.29, + "Plan Rows": 65139, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.01, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'fb372790-4b73-5a45-c97c-7e4287fa05be'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.007+00'::timestamp with time zone, '9cc0c1db-51d1-5bf2-e5a4-83cb93303974'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.021 + } + }, + { + "variant": "history", + "chat_size": 70000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 13.82, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.014, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 11922.0, + "Plan Rows": 45372, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.01, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'fb372790-4b73-5a45-c97c-7e4287fa05be'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.035+00'::timestamp with time zone, '169ee7a7-ca0d-aa05-bd8c-a1eb275708e7'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.019 + } + }, + { + "variant": "history", + "chat_size": 70000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 21.74, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.014, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 10814.04, + "Plan Rows": 25860, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.01, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'fb372790-4b73-5a45-c97c-7e4287fa05be'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.063+00'::timestamp with time zone, '987963b5-9cc6-d246-e263-52373e7f4b96'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.019 + } + }, + { + "variant": "history", + "chat_size": 20000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 88.93, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.014, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 9665.82, + "Plan Rows": 5569, + "Plan Width": 160, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.072+00'::timestamp with time zone, '97b900c6-4a0c-f323-4d98-d97c50488136'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.018 + } + }, + { + "variant": "history", + "chat_size": 20000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 111.12, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.004, + "Actual Total Time": 0.013, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 8668.14, + "Plan Rows": 3993, + "Plan Width": 160, + "Actual Startup Time": 0.004, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.08+00'::timestamp with time zone, '9042bcb2-df86-667b-9e54-e6e6c0c7923a'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.016, + "Triggers": [], + "Execution Time": 0.018 + } + }, + { + "variant": "history", + "chat_size": 20000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 134.87, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.013, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 6322.47, + "Plan Rows": 2398, + "Plan Width": 160, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '096d532a-b923-0f13-8dc9-243f47b54ac3'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.088+00'::timestamp with time zone, '81028714-cb0a-8e0f-6d74-4c379351efc2'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.018 + } + }, + { + "variant": "history", + "chat_size": 9000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 171.67, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.013, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 2737.17, + "Plan Rows": 815, + "Plan Width": 160, + "Actual Startup Time": 0.003, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.0909+00'::timestamp with time zone, 'b4d17acc-f297-d5d5-746e-2ef777eaa364'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.018 + } + }, + { + "variant": "history", + "chat_size": 9000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 182.11, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.013, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 1767.48, + "Plan Rows": 496, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.0945+00'::timestamp with time zone, '64cc6094-35b0-52bc-2c33-655d810cbabc'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.019, + "Triggers": [], + "Execution Time": 0.019 + } + }, + { + "variant": "history", + "chat_size": 9000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 194.46, + "Plan Rows": 51, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.013, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 635.83, + "Plan Rows": 167, + "Plan Width": 160, + "Actual Startup Time": 0.004, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '35bbad77-7634-ca61-4fd8-5d37539c23b8'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.0981+00'::timestamp with time zone, '8de6f8f9-25ec-97e3-e3b7-59eac1d2fd4c'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.021, + "Triggers": [], + "Execution Time": 0.019 + } + }, + { + "variant": "history", + "chat_size": 1000, + "position": 0.1, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 39.4, + "Plan Rows": 9, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.014, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 39.4, + "Plan Rows": 9, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.0991+00'::timestamp with time zone, 'a1580a3c-2826-e768-e4ea-f6c5a0e7171b'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.02, + "Triggers": [], + "Execution Time": 0.019 + } + }, + { + "variant": "history", + "chat_size": 1000, + "position": 0.5, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 23.92, + "Plan Rows": 5, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.014, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 23.92, + "Plan Rows": 5, + "Plan Width": 160, + "Actual Startup Time": 0.005, + "Actual Total Time": 0.009, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.0995+00'::timestamp with time zone, '86263544-3ca9-88ac-6a18-01fe3ed3d579'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.018, + "Triggers": [], + "Execution Time": 0.018 + } + }, + { + "variant": "history", + "chat_size": 1000, + "position": 0.9, + "plan": { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 8.44, + "Plan Rows": 1, + "Plan Width": 160, + "Actual Startup Time": 0.004, + "Actual Total Time": 0.012, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history_idx", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.42, + "Total Cost": 8.44, + "Plan Rows": 1, + "Plan Width": 160, + "Actual Startup Time": 0.004, + "Actual Total Time": 0.008, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = 'a3c16613-13f1-93e3-af30-521782e6eb21'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.0999+00'::timestamp with time zone, 'dc2ec078-4b31-5bfc-d9ac-d4e6671f761b'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 5, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 0, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.017, + "Triggers": [], + "Execution Time": 0.017 + } + } +] \ No newline at end of file diff --git a/evidence/research.json b/evidence/research.json new file mode 100644 index 0000000..5074a7a --- /dev/null +++ b/evidence/research.json @@ -0,0 +1,1114 @@ +{ + "postgres": "PostgreSQL 17.6 (Debian 17.6-2.pgdg12+1) on aarch64-unknown-linux-gnu, compiled by gcc (Debian 12.2.0-14+deb12u1) 12.2.0, 64-bit", + "messages": 100000, + "chat_sizes": [ + 70000, + 20000, + 9000, + 1000 + ], + "repeats": 3, + "warmup_per_case": 1, + "measurements_per_case": 5, + "page_size": 51, + "query": "SELECT * FROM messages WHERE chat_id=%s AND (created_at, id)>(%s, %s)\n ORDER BY created_at, id LIMIT %s", + "summary": { + "chat_only": { + "insert_100000_median_ms": 1386.7587500000127, + "index_bytes": 671744, + "wal_bytes_median": 38822560, + "queries": 180, + "errors": 0, + "concurrency": 1, + "p50_ms": 1.6852079999978287, + "p95_ms": 7.566457999985232, + "p99_ms": 46.40233399999261, + "successful_sql_rps": 282.01364182715855 + }, + "history": { + "insert_100000_median_ms": 1404.4910420000178, + "index_bytes": 7315456, + "wal_bytes_median": 43108512, + "queries": 180, + "errors": 0, + "concurrency": 1, + "p50_ms": 0.22610399997802233, + "p95_ms": 0.29704099995342403, + "p99_ms": 0.3384169999662845, + "successful_sql_rps": 4322.810483886352 + } + }, + "trials": [ + { + "variant": "chat_only", + "repeat": 1, + "insert_100000_ms": 1386.7587500000127, + "wal_bytes": 38822584, + "index_bytes": 671744, + "queries": [ + { + "chat_size": 70000, + "position": 0.1, + "latency_ms": [ + 6.914874999949916, + 6.668166999986624, + 6.523834000006445, + 6.556250000016917, + 6.811083000002327 + ], + "explain_execution_ms": 8.532 + }, + { + "chat_size": 70000, + "position": 0.5, + "latency_ms": [ + 46.40233399999261, + 6.385249999993903, + 6.055750000030002, + 7.862708000004659, + 7.713500000022577 + ], + "explain_execution_ms": 7.042 + }, + { + "chat_size": 70000, + "position": 0.9, + "latency_ms": [ + 6.462749999968764, + 6.372040999963247, + 6.377625000027365, + 6.360333999964496, + 6.293291999952544 + ], + "explain_execution_ms": 27.257 + }, + { + "chat_size": 20000, + "position": 0.1, + "latency_ms": [ + 2.6799170000231243, + 2.6539589999856616, + 2.6237499999979264, + 2.6159999999890715, + 2.5921669999888763 + ], + "explain_execution_ms": 3.549 + }, + { + "chat_size": 20000, + "position": 0.5, + "latency_ms": [ + 2.2941670000022896, + 2.2702500000377768, + 2.252708000014536, + 2.271042000018042, + 2.3105829999963134 + ], + "explain_execution_ms": 2.66 + }, + { + "chat_size": 20000, + "position": 0.9, + "latency_ms": [ + 1.9595829999730086, + 1.9207080000001042, + 2.059125000016593, + 2.0110829999566704, + 2.0071669999879305 + ], + "explain_execution_ms": 1.827 + }, + { + "chat_size": 9000, + "position": 0.1, + "latency_ms": [ + 1.3095409999550611, + 1.3007080000306814, + 1.288000000045031, + 1.280042000018966, + 1.2995419999697333 + ], + "explain_execution_ms": 1.592 + }, + { + "chat_size": 9000, + "position": 0.5, + "latency_ms": [ + 1.156374999993659, + 1.1407089999693198, + 1.144167000006746, + 1.1433750000264808, + 1.1405409999838412 + ], + "explain_execution_ms": 1.222 + }, + { + "chat_size": 9000, + "position": 0.9, + "latency_ms": [ + 1.0064999999599422, + 1.003166999964833, + 1.002000000028147, + 0.9964999999851898, + 0.9997909999697185 + ], + "explain_execution_ms": 0.846 + }, + { + "chat_size": 1000, + "position": 0.1, + "latency_ms": [ + 0.3318340000078024, + 0.34162500003276364, + 0.3277920000073209, + 0.32820799998489747, + 0.33399999995253893 + ], + "explain_execution_ms": 0.204 + }, + { + "chat_size": 1000, + "position": 0.5, + "latency_ms": [ + 0.31758300002593387, + 0.31320800002276883, + 0.3052920000072845, + 0.32208299995772904, + 0.31487500001503577 + ], + "explain_execution_ms": 0.151 + }, + { + "chat_size": 1000, + "position": 0.9, + "latency_ms": [ + 0.29320900000584516, + 0.29254200001105346, + 0.30541699999275806, + 0.2934999999979482, + 0.2922080000189453 + ], + "explain_execution_ms": 0.106 + } + ] + }, + { + "variant": "history", + "repeat": 1, + "insert_100000_ms": 1404.4910420000178, + "wal_bytes": 43108512, + "index_bytes": 7315456, + "queries": [ + { + "chat_size": 70000, + "position": 0.1, + "latency_ms": [ + 0.3173750000087239, + 0.29504200000474157, + 0.28329200000598576, + 0.3037079999899106, + 0.31616600000461403 + ], + "explain_execution_ms": 0.021 + }, + { + "chat_size": 70000, + "position": 0.5, + "latency_ms": [ + 0.2926249999859465, + 0.21762500000477303, + 0.19337500003757668, + 0.21075000000791988, + 0.23008300001947646 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 70000, + "position": 0.9, + "latency_ms": [ + 0.2326670000343256, + 0.22991600002342238, + 0.25349999998525163, + 0.2272909999874173, + 0.2270420000058948 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 20000, + "position": 0.1, + "latency_ms": [ + 0.2562920000173108, + 0.23475000000416912, + 0.22645799998599614, + 0.22308400002657436, + 0.2378749999820684 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 20000, + "position": 0.5, + "latency_ms": [ + 0.22491599997920275, + 0.25087500000609, + 0.22708300002705073, + 0.2274999999940519, + 0.22325000003320383 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 20000, + "position": 0.9, + "latency_ms": [ + 0.2279589999716336, + 0.22745899997289598, + 0.22599999999783904, + 0.2297500000167929, + 0.22641599997541562 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 9000, + "position": 0.1, + "latency_ms": [ + 0.24199999995744292, + 0.2252499999713109, + 0.22495799998978327, + 0.22799999999278953, + 0.2394169999888618 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 9000, + "position": 0.5, + "latency_ms": [ + 0.2292909999823678, + 0.3384169999662845, + 0.28125000000045475, + 0.24829200003750884, + 0.23558299994874687 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 9000, + "position": 0.9, + "latency_ms": [ + 0.2306659999931071, + 0.2268749999529973, + 0.29704099995342403, + 0.24354200002107973, + 0.23233300004221746 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 1000, + "position": 0.1, + "latency_ms": [ + 0.2479999999991378, + 0.2283750000060536, + 0.22741699996231546, + 0.2274999999940519, + 0.24287499996944462 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 1000, + "position": 0.5, + "latency_ms": [ + 0.22462499998709973, + 0.24941599997418962, + 0.2247919999831538, + 0.22466699999768025, + 0.2243339999949967 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 1000, + "position": 0.9, + "latency_ms": [ + 0.22287500001993976, + 0.2249590000360513, + 0.22291699997367687, + 0.2342500000054315, + 0.23004100000889594 + ], + "explain_execution_ms": 0.017 + } + ] + }, + { + "variant": "history", + "repeat": 2, + "insert_100000_ms": 1417.234876000009, + "wal_bytes": 43108512, + "index_bytes": 7315456, + "queries": [ + { + "chat_size": 70000, + "position": 0.1, + "latency_ms": [ + 0.3070409999850199, + 0.2651249999985339, + 0.2804580000201895, + 0.2466660000095544, + 0.4540420000012091 + ], + "explain_execution_ms": 0.021 + }, + { + "chat_size": 70000, + "position": 0.5, + "latency_ms": [ + 0.30112500002132947, + 0.23020800000495, + 0.24920800001382304, + 0.26024999999663123, + 0.23162500002626984 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 70000, + "position": 0.9, + "latency_ms": [ + 0.2297500000167929, + 0.22941700001410936, + 0.2268339999886848, + 0.2509999999915635, + 0.2265840000177377 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 20000, + "position": 0.1, + "latency_ms": [ + 0.24954200000593119, + 0.22820899999942412, + 0.22675000002436718, + 0.22470900000826077, + 0.22833299999547307 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 20000, + "position": 0.5, + "latency_ms": [ + 0.2232080000226233, + 0.22866699998758122, + 0.23858299999801602, + 0.22420799996325513, + 0.22699999999531428 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 20000, + "position": 0.9, + "latency_ms": [ + 0.2425420000236045, + 0.22604199995157614, + 0.22554200000968194, + 0.22283300000935924, + 0.24158299999044175 + ], + "explain_execution_ms": 0.025 + }, + { + "chat_size": 9000, + "position": 0.1, + "latency_ms": [ + 0.23279099997353114, + 0.23029199996926764, + 0.32120800000257077, + 0.26729199998953845, + 0.2517920000286722 + ], + "explain_execution_ms": 0.019 + }, + { + "chat_size": 9000, + "position": 0.5, + "latency_ms": [ + 0.23408300000937743, + 0.2288339999836353, + 0.23687499998459316, + 0.2249590000360513, + 0.2247919999831538 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 9000, + "position": 0.9, + "latency_ms": [ + 0.2243749999593092, + 0.22408299997778158, + 0.22312500004773028, + 0.23924999999280772, + 0.22150000000920045 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 1000, + "position": 0.1, + "latency_ms": [ + 0.22354100002530686, + 0.2302089999943746, + 0.22079199999325283, + 0.2222910000000411, + 0.2202919999945152 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 1000, + "position": 0.5, + "latency_ms": [ + 0.22287500001993976, + 0.22287499996309634, + 0.22041700003683218, + 0.2310830000169517, + 0.22616600000446851 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 1000, + "position": 0.9, + "latency_ms": [ + 0.22241599998551465, + 0.22308299998030634, + 0.22195799999735755, + 0.21995800000240706, + 0.21950000001424996 + ], + "explain_execution_ms": 0.017 + } + ] + }, + { + "variant": "chat_only", + "repeat": 2, + "insert_100000_ms": 1390.132249999965, + "wal_bytes": 38822560, + "index_bytes": 671744, + "queries": [ + { + "chat_size": 70000, + "position": 0.1, + "latency_ms": [ + 7.372583000005761, + 6.758875000002718, + 6.808458000023165, + 6.906250000042746, + 7.094250000022839 + ], + "explain_execution_ms": 40.541 + }, + { + "chat_size": 70000, + "position": 0.5, + "latency_ms": [ + 6.415666999998848, + 6.257875000017066, + 6.178666999971938, + 8.023291000029076, + 7.650417000036214 + ], + "explain_execution_ms": 7.198 + }, + { + "chat_size": 70000, + "position": 0.9, + "latency_ms": [ + 42.71466599999485, + 6.7938749999711945, + 6.601541000009092, + 6.471875000045202, + 6.461500000000342 + ], + "explain_execution_ms": 5.685 + }, + { + "chat_size": 20000, + "position": 0.1, + "latency_ms": [ + 2.8667090000453754, + 2.8354170000284284, + 2.8501250000090295, + 2.787916999977824, + 2.8024999999729516 + ], + "explain_execution_ms": 3.787 + }, + { + "chat_size": 20000, + "position": 0.5, + "latency_ms": [ + 2.3539160000041193, + 2.3988750000398795, + 2.3642079999603993, + 2.349291000030007, + 2.33629199999541 + ], + "explain_execution_ms": 2.82 + }, + { + "chat_size": 20000, + "position": 0.9, + "latency_ms": [ + 1.9656659999895965, + 1.9535420000238446, + 1.9440419999909864, + 1.940166999986559, + 1.9436249999671418 + ], + "explain_execution_ms": 1.83 + }, + { + "chat_size": 9000, + "position": 0.1, + "latency_ms": [ + 1.3667079999777343, + 1.36754199996858, + 1.3556669999843507, + 1.3793749999990723, + 1.357499999983247 + ], + "explain_execution_ms": 1.715 + }, + { + "chat_size": 9000, + "position": 0.5, + "latency_ms": [ + 1.1639999999601969, + 1.1638330000209862, + 1.1650000000145155, + 1.1809580000203823, + 1.1649159999933545 + ], + "explain_execution_ms": 1.275 + }, + { + "chat_size": 9000, + "position": 0.9, + "latency_ms": [ + 0.9753749999958927, + 0.9995409999987714, + 1.014125000040167, + 0.9871670000052291, + 0.9951249999744505 + ], + "explain_execution_ms": 0.854 + }, + { + "chat_size": 1000, + "position": 0.1, + "latency_ms": [ + 0.3327910000052725, + 0.3331660000185366, + 0.3437500000131877, + 0.3285419999770056, + 0.3382080000164933 + ], + "explain_execution_ms": 0.22 + }, + { + "chat_size": 1000, + "position": 0.5, + "latency_ms": [ + 0.3201250000302025, + 0.33404200001996287, + 0.31708299997035283, + 0.31666699999277625, + 0.3300420000300619 + ], + "explain_execution_ms": 0.156 + }, + { + "chat_size": 1000, + "position": 0.9, + "latency_ms": [ + 0.3005840000014359, + 0.28595899999572794, + 0.2849999999625652, + 0.2990409999483745, + 0.2847920000021986 + ], + "explain_execution_ms": 0.106 + } + ] + }, + { + "variant": "chat_only", + "repeat": 3, + "insert_100000_ms": 1330.3022509999778, + "wal_bytes": 38822560, + "index_bytes": 671744, + "queries": [ + { + "chat_size": 70000, + "position": 0.1, + "latency_ms": [ + 48.38283300000512, + 7.059540999989622, + 6.774750000033691, + 7.226040999967154, + 7.049875000006978 + ], + "explain_execution_ms": 8.497 + }, + { + "chat_size": 70000, + "position": 0.5, + "latency_ms": [ + 5.816499999980351, + 5.940875000021606, + 52.47504099997968, + 7.940541999971629, + 7.566457999985232 + ], + "explain_execution_ms": 6.953 + }, + { + "chat_size": 70000, + "position": 0.9, + "latency_ms": [ + 6.405041999983041, + 6.38737500003117, + 6.312415999957466, + 6.295207999983177, + 6.2736250000057225 + ], + "explain_execution_ms": 5.548 + }, + { + "chat_size": 20000, + "position": 0.1, + "latency_ms": [ + 2.6689579999583657, + 2.6040839999836862, + 2.6041659999691547, + 2.593499999989035, + 2.596417000006568 + ], + "explain_execution_ms": 11.285 + }, + { + "chat_size": 20000, + "position": 0.5, + "latency_ms": [ + 2.3674579999806156, + 2.497582999978931, + 2.4268750000260297, + 2.543584000022747, + 2.3092499999961547 + ], + "explain_execution_ms": 2.655 + }, + { + "chat_size": 20000, + "position": 0.9, + "latency_ms": [ + 1.982666000003519, + 1.9526250000012624, + 1.9421659999920848, + 2.0057080000128735, + 1.990875000046799 + ], + "explain_execution_ms": 1.878 + }, + { + "chat_size": 9000, + "position": 0.1, + "latency_ms": [ + 1.4241659999925105, + 1.449707999995553, + 1.3514999999983957, + 1.3168329999757589, + 1.3098750000040127 + ], + "explain_execution_ms": 1.607 + }, + { + "chat_size": 9000, + "position": 0.5, + "latency_ms": [ + 1.1759579999761627, + 1.1762500000145337, + 1.1795839999990676, + 1.1871670000118684, + 1.1549170000080267 + ], + "explain_execution_ms": 1.221 + }, + { + "chat_size": 9000, + "position": 0.9, + "latency_ms": [ + 0.9988330000396672, + 1.0168339999836462, + 1.0056670000153645, + 0.9815419999767983, + 0.9922919999780788 + ], + "explain_execution_ms": 0.831 + }, + { + "chat_size": 1000, + "position": 0.1, + "latency_ms": [ + 0.33158299999058727, + 0.32433300003731347, + 0.3312079999773232, + 0.3266250000137916, + 0.3368750000163345 + ], + "explain_execution_ms": 0.194 + }, + { + "chat_size": 1000, + "position": 0.5, + "latency_ms": [ + 0.3318330000183778, + 0.3219579999722555, + 0.3167920000350932, + 0.317624999979671, + 0.3145420000123522 + ], + "explain_execution_ms": 0.154 + }, + { + "chat_size": 1000, + "position": 0.9, + "latency_ms": [ + 0.29337500001247463, + 0.29699999998911153, + 0.3068749999783904, + 0.309750000042186, + 0.29841700001043137 + ], + "explain_execution_ms": 0.107 + } + ] + }, + { + "variant": "history", + "repeat": 3, + "insert_100000_ms": 1366.9856260000302, + "wal_bytes": 43108536, + "index_bytes": 7315456, + "queries": [ + { + "chat_size": 70000, + "position": 0.1, + "latency_ms": [ + 0.29145799999241717, + 0.25487500005283437, + 0.24249999995618055, + 0.24029200000086348, + 0.40070800002922624 + ], + "explain_execution_ms": 0.02 + }, + { + "chat_size": 70000, + "position": 0.5, + "latency_ms": [ + 0.24608299997908034, + 0.2166659999716103, + 0.19137499998578278, + 0.20683299999291194, + 0.18729199996414536 + ], + "explain_execution_ms": 0.034 + }, + { + "chat_size": 70000, + "position": 0.9, + "latency_ms": [ + 0.1954580000074202, + 0.19033400002399503, + 0.1846659999955591, + 0.20504200000459605, + 0.1987920000487975 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 20000, + "position": 0.1, + "latency_ms": [ + 0.1877920000197264, + 0.2019580000478527, + 0.1839579999796115, + 0.18112499998323983, + 0.18541700001151185 + ], + "explain_execution_ms": 0.017 + }, + { + "chat_size": 20000, + "position": 0.5, + "latency_ms": [ + 0.19970799996826827, + 0.18504199999824777, + 0.18249999999397915, + 0.17937500001607987, + 0.1814999999965039 + ], + "explain_execution_ms": 0.016 + }, + { + "chat_size": 20000, + "position": 0.9, + "latency_ms": [ + 0.18754100000251128, + 0.18725000001040826, + 0.18091699996602983, + 0.22283300000935924, + 0.20837499999970532 + ], + "explain_execution_ms": 0.017 + }, + { + "chat_size": 9000, + "position": 0.1, + "latency_ms": [ + 0.1893749999908323, + 0.20050000000537693, + 0.18170799995687048, + 0.1799160000359734, + 0.19962499999337524 + ], + "explain_execution_ms": 0.018 + }, + { + "chat_size": 9000, + "position": 0.5, + "latency_ms": [ + 0.2502920000324593, + 0.22045800000114468, + 0.21791599999687605, + 0.2180409999823496, + 0.23691699999517368 + ], + "explain_execution_ms": 0.017 + }, + { + "chat_size": 9000, + "position": 0.9, + "latency_ms": [ + 0.21720799998092843, + 0.21795899999688118, + 0.21658399998614186, + 0.2168750000350883, + 0.21812500000351065 + ], + "explain_execution_ms": 0.017 + }, + { + "chat_size": 1000, + "position": 0.1, + "latency_ms": [ + 0.21791700004314407, + 0.2171250000060354, + 0.21858299999166775, + 0.21612500000856016, + 0.22783299999673545 + ], + "explain_execution_ms": 0.017 + }, + { + "chat_size": 1000, + "position": 0.5, + "latency_ms": [ + 0.2180000000180371, + 0.21766600002592895, + 0.21808299999293013, + 0.2169169999888254, + 0.2186660000234042 + ], + "explain_execution_ms": 0.017 + }, + { + "chat_size": 1000, + "position": 0.9, + "latency_ms": [ + 0.21783400001140762, + 0.21633299996892674, + 0.2234579999935704, + 0.21850000001677472, + 0.2274999999940519 + ], + "explain_execution_ms": 0.017 + } + ] + } + ], + "snapshot": { + "counts": { + "users": 10, + "chats": 4, + "chat_members": 8, + "messages": 100000, + "schema_migrations": 1 + }, + "messages_sha256": "36958977cf7f00bc537cb8e2b0052fd83cc8358d0d44d72304d0856d1cf43870", + "samples": [ + { + "chat_id": "fb372790-4b73-5a45-c97c-7e4287fa05be", + "created_at": "2026-01-01 00:00:00.000001+00:00", + "id": "0406791b-bf11-c4be-113d-e56c41008a52", + "sender_id": "fbb798c2-5241-0201-2e7b-d22eef00e52d", + "text": "Synthetic message 1 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "fb372790-4b73-5a45-c97c-7e4287fa05be", + "created_at": "2026-01-01 00:00:00.035000+00:00", + "id": "169ee7a7-ca0d-aa05-bd8c-a1eb275708e7", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 35000 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "fb372790-4b73-5a45-c97c-7e4287fa05be", + "created_at": "2026-01-01 00:00:00.070000+00:00", + "id": "dbf177a7-e559-4eb5-2f16-d7594a0c375b", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 70000 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "096d532a-b923-0f13-8dc9-243f47b54ac3", + "created_at": "2026-01-01 00:00:00.070001+00:00", + "id": "042eb2e1-c673-28b2-57e6-bbc531f9f886", + "sender_id": "fbb798c2-5241-0201-2e7b-d22eef00e52d", + "text": "Synthetic message 70001 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "096d532a-b923-0f13-8dc9-243f47b54ac3", + "created_at": "2026-01-01 00:00:00.080000+00:00", + "id": "9042bcb2-df86-667b-9e54-e6e6c0c7923a", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 80000 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "096d532a-b923-0f13-8dc9-243f47b54ac3", + "created_at": "2026-01-01 00:00:00.090000+00:00", + "id": "5409a1ee-b7e3-f2fa-1b87-173c59924f20", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 90000 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "35bbad77-7634-ca61-4fd8-5d37539c23b8", + "created_at": "2026-01-01 00:00:00.090001+00:00", + "id": "8a98c150-4c98-8f1a-478f-2d01e8725584", + "sender_id": "fbb798c2-5241-0201-2e7b-d22eef00e52d", + "text": "Synthetic message 90001 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "35bbad77-7634-ca61-4fd8-5d37539c23b8", + "created_at": "2026-01-01 00:00:00.094500+00:00", + "id": "64cc6094-35b0-52bc-2c33-655d810cbabc", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 94500 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "35bbad77-7634-ca61-4fd8-5d37539c23b8", + "created_at": "2026-01-01 00:00:00.099000+00:00", + "id": "26f56eb2-d617-f971-efa9-fc332ae1272b", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 99000 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "a3c16613-13f1-93e3-af30-521782e6eb21", + "created_at": "2026-01-01 00:00:00.099001+00:00", + "id": "7d565f77-f911-6eb9-62a9-dcf414afb5f3", + "sender_id": "fbb798c2-5241-0201-2e7b-d22eef00e52d", + "text": "Synthetic message 99001 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "a3c16613-13f1-93e3-af30-521782e6eb21", + "created_at": "2026-01-01 00:00:00.099500+00:00", + "id": "86263544-3ca9-88ac-6a18-01fe3ed3d579", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 99500 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "chat_id": "a3c16613-13f1-93e3-af30-521782e6eb21", + "created_at": "2026-01-01 00:00:00.100000+00:00", + "id": "103bbef2-c217-19c4-d209-a1c1cdd3a1e2", + "sender_id": "bdb1dd10-5679-979c-a82b-28edd1c8ccd2", + "text": "Synthetic message 100000 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + } + ] + }, + "utc": "2026-10-22T11:09:43.745407+00:00", + "project": "lab2-research-85021a09d9", + "command": "./scripts/research.sh", + "host": "Darwin arm64", + "commit": "9961362 + working tree", + "postgres_limits": { + "cpus": 1, + "memory_mib": 512 + }, + "api_runner_limits": { + "cpus": 1, + "memory_mib": 256 + }, + "source": { + "container": "c0b5e13dc5d2fd8bbd488a9b227eb62e4fa6a3896f6b28cf464b94e3d8f9032f", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "volumes": [ + "lab2-research-85021a09d9_postgres_data" + ] + }, + "restored": { + "container": "1c23bc29d590048481d8e2d18dfe340627309f20f004d4ffdb8892c3582fb8f1", + "image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3", + "volumes": [ + "lab2-research-85021a09d9_restore_data" + ] + }, + "docker": "29.0.1 aarch64 CPUs=10 RAM=8217448448", + "stats_after_queries": "{\"BlockIO\":\"131kB / 583MB\",\"CPUPerc\":\"0.05%\",\"Container\":\"c0b5e13dc5d2fd8bbd488a9b227eb62e4fa6a3896f6b28cf464b94e3d8f9032f\",\"ID\":\"c0b5e13dc5d2\",\"MemPerc\":\"12.35%\",\"MemUsage\":\"63.24MiB / 512MiB\",\"Name\":\"lab2-research-85021a09d9-postgres-1\",\"NetIO\":\"378kB / 33.3MB\",\"PIDs\":\"6\"}", + "backup": { + "file": "backups/lab2-research-85021a09d9.dump", + "bytes": 3067615, + "seconds": 0.5438075830461457 + }, + "restore": { + "seconds": 0.6752373329363763, + "verification_seconds": 2.342134208069183, + "verification": { + "counts": { + "users": 10, + "chats": 4, + "chat_members": 8, + "messages": 100000, + "schema_migrations": 1 + }, + "messages_sha256": "36958977cf7f00bc537cb8e2b0052fd83cc8358d0d44d72304d0856d1cf43870", + "samples_verified": 12, + "all_messages_match": true, + "after_backup_message_missing_in_restore": true, + "source_messages": 100001 + }, + "post_backup_insert": { + "id": "9b10784e-e462-01b4-6520-1c572ae115d9", + "source_messages": 100001 + } + }, + "total_seconds": 22.895743250031956 +} \ No newline at end of file diff --git a/evidence/research.txt b/evidence/research.txt new file mode 100644 index 0000000..ef6bb5e --- /dev/null +++ b/evidence/research.txt @@ -0,0 +1,32 @@ +Research project: lab2-research-85021a09d9 +Measuring two indexes on the same 100000 messages, three repeats... +{ + "chat_only": { + "insert_100000_median_ms": 1386.7587500000127, + "index_bytes": 671744, + "wal_bytes_median": 38822560, + "queries": 180, + "errors": 0, + "concurrency": 1, + "p50_ms": 1.6852079999978287, + "p95_ms": 7.566457999985232, + "p99_ms": 46.40233399999261, + "successful_sql_rps": 282.01364182715855 + }, + "history": { + "insert_100000_median_ms": 1404.4910420000178, + "index_bytes": 7315456, + "wal_bytes_median": 43108512, + "queries": 180, + "errors": 0, + "concurrency": 1, + "p50_ms": 0.22610399997802233, + "p95_ms": 0.29704099995342403, + "p99_ms": 0.3384169999662845, + "successful_sql_rps": 4322.810483886352 + } +} +PASS: backup 0.54s; restore 0.68s; 100000 messages, all hashes and 12 samples match +PASS: restored into a different container and fresh volume; one post-backup commit is absent as expected +Results: evidence/research.json, evidence/query-plans.json; dump stays outside Git in backups/ +Removed only temporary research project: lab2-research-85021a09d9 diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..c65e647 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,618 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger — Lab 6: signals and SLO", + "schemaVersion": 41, + "version": 1, + "refresh": "5s", + "time": { + "from": "now-30m", + "to": "now" + }, + "tags": [ + "lab6", + "slo" + ], + "panels": [ + { + "id": 1, + "title": "Business API RPS", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "description": "", + "targets": [ + { + "refId": "A", + "expr": "sum by (method, route) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{method}} {{route}}" + } + ] + }, + { + "id": 2, + "title": "API 5xx / all business requests", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "description": "", + "targets": [ + { + "refId": "A", + "expr": "lab6:api_bad_ratio:1m", + "legendFormat": "API 5xx / all business requests" + } + ] + }, + { + "id": 3, + "title": "API latency: aggregated histogram", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "description": "Seconds to response headers; excludes health. Sum buckets across replicas before percentile.", + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))", + "legendFormat": "p99" + } + ] + }, + { + "id": 4, + "title": "Time from acceptance to ready: completed jobs", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "description": "Success histogram only. SLO below also includes jobs that never complete.", + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_ready_duration_seconds_bucket[2m])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_duration_seconds_bucket[2m])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_ready_duration_seconds_bucket[2m])))", + "legendFormat": "p99" + } + ] + }, + { + "id": 5, + "title": "Job attempts by outcome", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "description": "", + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (rate(messenger_job_attempts_total[1m]))", + "legendFormat": "{{outcome}}" + } + ] + }, + { + "id": 6, + "title": "Worker processing p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "description": "", + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(messenger_job_duration_seconds_bucket[1m])))", + "legendFormat": "{{outcome}}" + } + ] + }, + { + "id": 7, + "title": "Outstanding jobs (DB source of truth)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "description": "Queued + processing durable jobs, including retry/outbox; not Redis XLEN.", + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_depth)", + "legendFormat": "Outstanding jobs (DB source of truth)" + } + ] + }, + { + "id": 8, + "title": "Oldest unfinished job age", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "description": "", + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_oldest_age_seconds)", + "legendFormat": "Oldest unfinished job age" + } + ] + }, + { + "id": 9, + "title": "DB pool used / waiting", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "description": "", + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name, state) (messenger_db_pool)", + "legendFormat": "{{service_name}} {{state}}" + } + ] + }, + { + "id": 10, + "title": "Worker busy / capacity", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "description": "", + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_worker_busy) or vector(0)", + "legendFormat": "busy" + }, + { + "refId": "B", + "expr": "sum(messenger_worker_capacity) or vector(0)", + "legendFormat": "capacity" + } + ] + }, + { + "id": 11, + "title": "Process CPU: cores consumed", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "description": "", + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name) (rate(messenger_process_cpu_time_seconds_total[1m]))", + "legendFormat": "{{service_name}}" + } + ] + }, + { + "id": 12, + "title": "Process RSS", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "description": "", + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name) (messenger_process_memory_bytes)", + "legendFormat": "{{service_name}}" + } + ] + }, + { + "id": 13, + "title": "API availability SLO: >= 99% / 1h", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "description": "5xx bad; all instrumented business requests denominator; 4xx not server failures.", + "targets": [ + { + "refId": "A", + "expr": "1-lab6:api_bad_ratio:1h", + "legendFormat": "API availability SLO: >= 99% / 1h" + } + ] + }, + { + "id": 14, + "title": "Ready SLO: >= 95% within 30s / 1h", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "description": "Acceptance cohorts mature after 30s. Failed, stuck and late jobs are BAD. No ready-only bias.", + "targets": [ + { + "refId": "A", + "expr": "1-lab6:ready_bad_ratio:1h", + "legendFormat": "Ready SLO: >= 95% within 30s / 1h" + } + ] + }, + { + "id": 15, + "title": "API burn rate (budget / hour)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 56, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "description": "Fast: 2m AND 30s > 6x, for 15s. Slow: 6m AND 1m > 3x, for 30s.", + "targets": [ + { + "refId": "A", + "expr": "lab6:api_bad_ratio:30s/0.01", + "legendFormat": "30s" + }, + { + "refId": "B", + "expr": "lab6:api_bad_ratio:2m/0.01", + "legendFormat": "2m" + }, + { + "refId": "C", + "expr": "lab6:api_bad_ratio:1m/0.01", + "legendFormat": "1m" + }, + { + "refId": "D", + "expr": "lab6:api_bad_ratio:6m/0.01", + "legendFormat": "6m" + } + ] + }, + { + "id": 16, + "title": "Ready burn rate (budget / hour)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 56, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "description": "Fast: 2m AND 30s > 6x, for 15s. Slow: 6m AND 1m > 3x, for 30s.", + "targets": [ + { + "refId": "A", + "expr": "lab6:ready_bad_ratio:30s/0.05", + "legendFormat": "30s" + }, + { + "refId": "B", + "expr": "lab6:ready_bad_ratio:2m/0.05", + "legendFormat": "2m" + }, + { + "refId": "C", + "expr": "lab6:ready_bad_ratio:1m/0.05", + "legendFormat": "1m" + }, + { + "refId": "D", + "expr": "lab6:ready_bad_ratio:6m/0.05", + "legendFormat": "6m" + } + ] + }, + { + "id": 17, + "title": "Local alert states", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 64, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "description": "", + "targets": [ + { + "refId": "A", + "expr": "ALERTS{alertstate=~\"pending|firing\"}", + "legendFormat": "{{alertname}} {{alertstate}}" + } + ] + }, + { + "id": 18, + "title": "API / worker structured logs (click TraceID)", + "type": "logs", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "x": 12, + "y": 64, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "{service_name=~\"messenger-api|messenger-worker\"} | json" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true + } + }, + { + "id": 19, + "title": "Recent real worker traces", + "type": "table", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "x": 0, + "y": 72, + "w": 24, + "h": 9 + }, + "targets": [ + { + "refId": "A", + "queryType": "traceql", + "query": "{ resource.service.name = \"messenger-worker\" }", + "limit": 20, + "tableType": "traces", + "filters": [] + } + ] + } + ], + "annotations": { + "list": [] + } +} diff --git a/infra/grafana/provisioning/dashboards/provider.yaml b/infra/grafana/provisioning/dashboards/provider.yaml new file mode 100644 index 0000000..731d8b7 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/provider.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 +providers: + - name: messenger + folder: Lab 6 + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/sources.yaml b/infra/grafana/provisioning/datasources/sources.yaml new file mode 100644 index 0000000..c623c46 --- /dev/null +++ b/infra/grafana/provisioning/datasources/sources.yaml @@ -0,0 +1,35 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 5s + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + matcherRegex: '"trace_id": "([a-f0-9]{32})"' + datasourceUid: tempo + url: '$${__value.raw}' + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogs: + datasourceUid: loki + spanStartTimeShift: '-1m' + spanEndTimeShift: '1m' + filterByTraceID: true + tags: [{key: service.name, value: service_name}] + nodeGraph: + enabled: true diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..c39b8d5 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,40 @@ +global + log stdout format raw local0 + maxconn 256 + +defaults + mode http + log global + option dontlognull + option http-server-close + timeout connect 1s + timeout client 40s + timeout server 60s + timeout http-request 5s + timeout check 1s + retries 1 + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold timeout 1s + hold refused 1s + hold other 1s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost + http-check expect status 200 + default-server inter 1s fastinter 1s downinter 1s fall 2 rise 2 + server-template replica 10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none diff --git a/infra/otel/alerts.yaml b/infra/otel/alerts.yaml new file mode 100644 index 0000000..c0edcb4 --- /dev/null +++ b/infra/otel/alerts.yaml @@ -0,0 +1,113 @@ +groups: +- name: messenger-slo + interval: 5s + rules: + - record: lab6:api_bad_ratio:30s + expr: (sum(rate(messenger_http_requests_total{status_class="5xx"}[30s])) or vector(0)) + / sum(rate(messenger_http_requests_total[30s])) + - record: lab6:ready_bad_ratio:30s + expr: sum(rate(messenger_ready_deadlines_total{outcome="bad"}[30s])) / sum(rate(messenger_ready_deadlines_total[30s])) + - record: lab6:api_bad_ratio:1m + expr: (sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) or vector(0)) + / sum(rate(messenger_http_requests_total[1m])) + - record: lab6:ready_bad_ratio:1m + expr: sum(rate(messenger_ready_deadlines_total{outcome="bad"}[1m])) / sum(rate(messenger_ready_deadlines_total[1m])) + - record: lab6:api_bad_ratio:2m + expr: (sum(rate(messenger_http_requests_total{status_class="5xx"}[2m])) or vector(0)) + / sum(rate(messenger_http_requests_total[2m])) + - record: lab6:ready_bad_ratio:2m + expr: sum(rate(messenger_ready_deadlines_total{outcome="bad"}[2m])) / sum(rate(messenger_ready_deadlines_total[2m])) + - record: lab6:api_bad_ratio:6m + expr: (sum(rate(messenger_http_requests_total{status_class="5xx"}[6m])) or vector(0)) + / sum(rate(messenger_http_requests_total[6m])) + - record: lab6:ready_bad_ratio:6m + expr: sum(rate(messenger_ready_deadlines_total{outcome="bad"}[6m])) / sum(rate(messenger_ready_deadlines_total[6m])) + - record: lab6:api_bad_ratio:1h + expr: (sum(rate(messenger_http_requests_total{status_class="5xx"}[1h])) or vector(0)) + / sum(rate(messenger_http_requests_total[1h])) + - record: lab6:ready_bad_ratio:1h + expr: sum(rate(messenger_ready_deadlines_total{outcome="bad"}[1h])) / sum(rate(messenger_ready_deadlines_total[1h])) + - alert: APIAvailabilityFastBurn + expr: (lab6:api_bad_ratio:2m > 6 * 0.01) and (lab6:api_bad_ratio:30s > 6 * 0.01) + and (sum(increase(messenger_http_requests_total[2m])) >= 30) and (sum(increase(messenger_http_requests_total[30s])) + >= 2) + for: 15s + labels: + severity: critical + slo: api + annotations: + summary: 'api: 6x budget burn in both 2m and 30s' + description: Budget 0.01; rolling SLO window 1h. Requires real traffic, not + health requests. + runbook_url: https://gitea.privcloud.tech/Savelev_Ivan/mtusi2026lab6/src/branch/main/docs/runbook.md#api + runbook: docs/runbook.md#api + - alert: APIAvailabilitySlowBurn + expr: (lab6:api_bad_ratio:6m > 3 * 0.01) and (lab6:api_bad_ratio:1m > 3 * 0.01) + and (sum(increase(messenger_http_requests_total[6m])) >= 30) and (sum(increase(messenger_http_requests_total[1m])) + >= 2) + for: 30s + labels: + severity: warning + slo: api + annotations: + summary: 'api: 3x budget burn in both 6m and 1m' + description: Budget 0.01; rolling SLO window 1h. Requires real traffic, not + health requests. + runbook_url: https://gitea.privcloud.tech/Savelev_Ivan/mtusi2026lab6/src/branch/main/docs/runbook.md#api + runbook: docs/runbook.md#api + - alert: AttachmentReadyFastBurn + expr: (lab6:ready_bad_ratio:2m > 6 * 0.05) and (lab6:ready_bad_ratio:30s > 6 * + 0.05) and (sum(increase(messenger_ready_deadlines_total[2m])) >= 5) and (sum(increase(messenger_ready_deadlines_total[30s])) + >= 2) + for: 15s + labels: + severity: critical + slo: ready + annotations: + summary: 'ready: 6x budget burn in both 2m and 30s' + description: Budget 0.05; rolling SLO window 1h. Requires real traffic, not + health requests. + runbook_url: https://gitea.privcloud.tech/Savelev_Ivan/mtusi2026lab6/src/branch/main/docs/runbook.md#ready + runbook: docs/runbook.md#ready + - alert: AttachmentReadySlowBurn + expr: (lab6:ready_bad_ratio:6m > 3 * 0.05) and (lab6:ready_bad_ratio:1m > 3 * + 0.05) and (sum(increase(messenger_ready_deadlines_total[6m])) >= 5) and (sum(increase(messenger_ready_deadlines_total[1m])) + >= 2) + for: 30s + labels: + severity: warning + slo: ready + annotations: + summary: 'ready: 3x budget burn in both 6m and 1m' + description: Budget 0.05; rolling SLO window 1h. Requires real traffic, not + health requests. + runbook_url: https://gitea.privcloud.tech/Savelev_Ivan/mtusi2026lab6/src/branch/main/docs/runbook.md#ready + runbook: docs/runbook.md#ready + - alert: APIHighErrorRate + expr: lab6:api_bad_ratio:30s > 0.05 + for: 5s + labels: + severity: warning + annotations: + summary: API 5xx > 5% for 30s (single-window comparator) + runbook: docs/runbook.md#api + runbook_url: https://gitea.privcloud.tech/Savelev_Ivan/mtusi2026lab6/src/branch/main/docs/runbook.md#api + - alert: AttachmentQueueLag + expr: (max(messenger_queue_oldest_age_seconds) > 20) and (max(messenger_queue_depth) + > 0) + for: 10s + labels: + severity: warning + annotations: + summary: Oldest unfinished job exceeds 20s; queue not empty + runbook: docs/runbook.md#ready + runbook_url: https://gitea.privcloud.tech/Savelev_Ivan/mtusi2026lab6/src/branch/main/docs/runbook.md#ready + - alert: TelemetryCollectorUnavailable + expr: up{job="collector"} == 0 + for: 15s + labels: + severity: warning + annotations: + summary: Collector scrape unavailable; SLI may be incomplete + runbook: docs/runbook.md#telemetry + runbook_url: https://gitea.privcloud.tech/Savelev_Ivan/mtusi2026lab6/src/branch/main/docs/runbook.md#telemetry diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..66700a3 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,74 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 32 + batch: + timeout: 1s + send_batch_size: 64 + send_batch_max_size: 128 + +exporters: + prometheus: + endpoint: 0.0.0.0:9464 + metric_expiration: 30s + resource_to_telemetry_conversion: + enabled: true + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 2s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 64 + retry_on_failure: + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 15s + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + timeout: 2s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 64 + retry_on_failure: + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 15s + +service: + extensions: [health_check] + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] diff --git a/infra/otel/loki.yaml b/infra/otel/loki.yaml new file mode 100644 index 0000000..33a2d4d --- /dev/null +++ b/infra/otel/loki.yaml @@ -0,0 +1,37 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 168h + otlp_config: + resource_attributes: + ignore_defaults: true + attributes_config: + - action: index_label + attributes: [service.name, deployment.environment.name] + - action: structured_metadata + attributes: [service.instance.id, service.version] +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem diff --git a/infra/otel/prometheus.yaml b/infra/otel/prometheus.yaml new file mode 100644 index 0000000..6de3ad1 --- /dev/null +++ b/infra/otel/prometheus.yaml @@ -0,0 +1,15 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: + - /etc/prometheus/alerts.yaml +scrape_configs: + - job_name: messenger + static_configs: + - targets: [otel-collector:9464] + - job_name: collector + static_configs: + - targets: [otel-collector:8888] + - job_name: prometheus + static_configs: + - targets: [localhost:9090] diff --git a/infra/otel/tempo.yaml b/infra/otel/tempo.yaml new file mode 100644 index 0000000..e1dd391 --- /dev/null +++ b/infra/otel/tempo.yaml @@ -0,0 +1,20 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 +ingester: + max_block_duration: 1m +compactor: + compaction: + block_retention: 168h +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..9dfbb86 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,30 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name varchar(100) NOT NULL CHECK (char_length(display_name) >= 1), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL CHECK (char_length(title) >= 1), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000 AND text !~ '^\s*$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_message_idempotency.sql b/migrations/002_message_idempotency.sql new file mode 100644 index 0000000..375b58c --- /dev/null +++ b/migrations/002_message_idempotency.sql @@ -0,0 +1,10 @@ +-- Добавочная миграция: существующие данные и таблицы lab2 сохраняются. +CREATE TABLE message_idempotency ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + key varchar(128) NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_text text NOT NULL, + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, key) +); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..5a9db48 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,34 @@ +-- Только добавления к lab3; старые сообщения и ключи сохраняются. +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL, + object_key text NOT NULL UNIQUE, + filename varchar(255) NOT NULL, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX attachments_chat_idx ON attachments(chat_id); + +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id), + position integer NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position) +); +ALTER TABLE message_idempotency + ADD COLUMN request_attachment_ids uuid[] NOT NULL DEFAULT '{}'; + +-- Фиксируется ДО S3 PUT. После COMMIT ready-метаданных удаляется атомарно с ними. +CREATE TABLE attachment_uploads ( + id uuid PRIMARY KEY, + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachment_uploads_created_idx ON attachment_uploads(created_at); diff --git a/migrations/004_attachment_jobs.sql b/migrations/004_attachment_jobs.sql new file mode 100644 index 0000000..dd53682 --- /dev/null +++ b/migrations/004_attachment_jobs.sql @@ -0,0 +1,42 @@ +-- Lab5: существующие ready-вложения lab4 остаются доступными. +ALTER TABLE attachments + ADD COLUMN status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued','processing','ready','failed')), + ADD COLUMN variants jsonb NOT NULL DEFAULT '[]', + ADD COLUMN error_code text, + ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(); +UPDATE attachments SET updated_at=created_at; + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version=1), + staging_key text NOT NULL UNIQUE, + attempt integer NOT NULL DEFAULT 1 CHECK (attempt>=1), + max_attempts integer NOT NULL CHECK (max_attempts>=1), + replay_count integer NOT NULL DEFAULT 0, + lease_token uuid, + lease_until timestamptz, + next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), + last_error text, + staging_deleted_at timestamptz, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs(lease_until) WHERE lease_until IS NOT NULL; + +CREATE TABLE attachment_outbox ( + job_id uuid NOT NULL REFERENCES attachment_jobs(id), + attempt integer NOT NULL, + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + PRIMARY KEY(job_id, attempt) +); +CREATE INDEX attachment_outbox_due_idx ON attachment_outbox(available_at) WHERE published_at IS NULL; + +CREATE TABLE attachment_job_events ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id), + attempt integer NOT NULL, + event text NOT NULL, + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/migrations/005_observability.sql b/migrations/005_observability.sql new file mode 100644 index 0000000..c8bb8ae --- /dev/null +++ b/migrations/005_observability.sql @@ -0,0 +1,9 @@ +-- Context belongs to the durable job, not the lifetime of the HTTP process. +ALTER TABLE attachment_jobs + ADD COLUMN traceparent text, + ADD COLUMN tracestate text, + ADD COLUMN ready_at timestamptz; +-- Existing completed jobs predate instrumentation; preserve their actual update time. +UPDATE attachment_jobs j SET ready_at=a.updated_at +FROM attachments a WHERE a.id=j.attachment_id AND a.status='ready'; +CREATE INDEX attachment_jobs_created_idx ON attachment_jobs(created_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..652ef5f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,50 @@ +# Основные библиотеки +fastapi==0.142.4 +uvicorn==0.54.0 +pydantic==2.13.5 + +# Их зависимости +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +click==8.5.0 +h11==0.16.0 +idna==3.20 +opentelemetry-api==1.45.1 +pydantic-core==2.46.5 +starlette==1.7.0 +typing-extensions==4.16.0 +typing-inspection==0.4.4 + +# PostgreSQL, сессии и Argon2id +psycopg[binary,pool]==3.3.6 +redis==8.1.0 +argon2-cffi==25.1.0 + +# Транзитивные зависимости новых библиотек +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +pycparser==3.11 + +# S3 и проверка настоящих PNG/JPEG (без изменения оригинала) +minio==7.2.20 +Pillow==13.0.0 +certifi==2026.10.23 +pycryptodome==4.3.0 +urllib3==2.8.0 + +# Explicit OpenTelemetry SDK (all signals), no automatic header/SQL/body capture. +opentelemetry-sdk==1.45.1 +opentelemetry-exporter-otlp-proto-http==1.45.1 +opentelemetry-semantic-conventions==0.66b1 +opentelemetry-exporter-otlp-proto-common==1.45.1 +opentelemetry-proto==1.45.1 +opentelemetry-exporter-http-transport==0.66b1 +opentelemetry-exporter-otlp-common==0.66b1 +googleapis-common-protos==1.76.0 +protobuf==7.36.2 +requests==2.34.2 +charset-normalizer==3.5.2 +psutil==7.2.2 diff --git a/research/README.lab2.md b/research/README.lab2.md new file mode 100644 index 0000000..a2f780a --- /dev/null +++ b/research/README.lab2.md @@ -0,0 +1,12 @@ +# Трек 2 на оценку 5 + +Запуск из корня: `./scripts/research-lab2.sh`. Нужны Docker Compose, Python 3 и интернет для первой сборки. Скрипт собирает API-образ и использует отдельный Compose project с новыми volumes. Текущий стенд не участвует. + +- run.py управляет контейнерами, pg_dump, pg_restore, файлами evidence и cleanup. +- benchmark.py выполняет SQL и сверку внутри API-образа. Перед заполнением требует пустую БД. Исследовательский флаг устанавливает только драйвер. + +Датасет: 100000 сообщений в 4 чатах (70000/20000/9000/1000). Сравниваются индекс chat_id и индекс (chat_id,created_at,id), по 3 повтора с одинаковыми ограничениями и записями. На каждый чат — страницы около 10/50/90%, один прогрев и пять измерений, 51 строка. Измеряется INSERT SELECT с COMMIT, WAL и размер индекса. EXPLAIN и основные цифры сохраняются в evidence. + +После dump создаётся ещё одно сообщение. Backup восстанавливается в другой контейнер и свежий volume. Сравниваются все счётчики, 12 полных записей и SHA-256 всех сообщений. Последующая запись ожидаемо отсутствует в восстановленном снимке. + +Файлы research.json, query-plans.json и research.txt обновляются автоматически. Dump остаётся в игнорируемом backups/. Временные исследовательские volumes удаляются после проверки. Методика, фактические результаты и ограничения лабораторной 2 описаны в [REPORT.lab2.md](../REPORT.lab2.md). diff --git a/research/README.md b/research/README.md new file mode 100644 index 0000000..85be75e --- /dev/null +++ b/research/README.md @@ -0,0 +1,37 @@ +# Лабораторная 6, оценка 5, трек 2: SLO и обнаружение инцидентов + +После `./scripts/start.sh`: `./scripts/research.sh` (около 11 минут). +Полный набор проверок вместе с сохранённым минимумом lab5: +`./scripts/check-lab6.sh` (около 17–20 минут при готовых образах). +Эксперимент намеренно блокирует messages и останавливает worker своего локального +синтетического стенда. Отказы ограничены по времени, worker восстанавливается +в `finally`. Нужны Docker Compose, Python 3.11 и `.venv` с корневыми requirements. + +Гипотеза: alert по двум окнам не сработает на 4s всплеск, но обнаружит длительную +деградацию БД и worker; после восстановления станет inactive. Сравнение — простой +5%/30s alert. Параметры заранее заданы в `infra/otel/alerts.yaml` и `docs/runbook.md`; +эксперимент их не меняет. + +Профиль: 2 POST сообщения/сек и 0.5 PNG upload/сек, fixed arrival, максимум +16 in-flight, без retries. Jobs реально обрабатываются через S3 и broker. +Baseline 120s → DB spike 4s → recovery 60s → DB lock 90s → recovery 100s → +worker stopped 120s → recovery 120s. Состояния alerts и backlog читаются каждые 5s +из Prometheus. SQL/Docker создают и снимают воздействие; причина определяется +по сохранённым metrics/logs/traces, а не по выводу команды внедрения отказа. + +Результаты: `evidence/lab6-slo-research.json` (каждый запрос без body/credentials, +фазы, detection/resolution, p50/p95/p99, RPS), `lab6-slo-series.json` +(CPU/RSS, pool, saturation и остальные временные ряды), +`lab6-db-incident-trace.json`, `lab6-db-incident-logs.json`, +`lab6-worker-incident-trace.json`, `lab6-worker-incident-logs.json`. +`student-tests/promtool-slo.yaml` дополнительно проверяет правила на повторяемых +синтетических временных рядах: всплеск, длительный отказ, recovery, stalled jobs, +нет трафика. Это дополнение к live эксперименту, не замена ему. + +Ограничения: один Docker-host, небольшие PNG, один live прогон каждого отказа; +шаг измерения alerts 5s, задержки SDK export/scrape. Часовой SLO задан, но +эксперимент короче часа: результат проверяет детектор, а не долгосрочную надёжность. +False-positive count нельзя переносить на другие нагрузки/окна. + +Историческое исследование lab2: `run.py`, `benchmark.py`, `scripts/research-lab2.sh`, +[README.lab2.md](README.lab2.md). diff --git a/research/__init__.py b/research/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/research/benchmark.py b/research/benchmark.py new file mode 100644 index 0000000..93b51fe --- /dev/null +++ b/research/benchmark.py @@ -0,0 +1,168 @@ +"""SQL-эксперимент запускается только в отдельном research Compose project.""" +from datetime import datetime, timedelta, timezone +import hashlib +import json +import os +import secrets +import statistics +import sys +import time +from uuid import UUID + +import psycopg +from psycopg.conninfo import make_conninfo +from psycopg.rows import dict_row + +from src import db, migrate, passwords +from src.config import secret + + +SIZES = [70000, 20000, 9000, 1000] +START = datetime(2026, 1, 1, tzinfo=timezone.utc) +QUERY = """SELECT * FROM messages WHERE chat_id=%s AND (created_at, id)>(%s, %s) + ORDER BY created_at, id LIMIT %s""" + + +def identifier(value): + return UUID(hashlib.md5(value.encode(), usedforsecurity=False).hexdigest()) + + +def connect(restored=False): + url = secret("DATABASE_URL") + if restored: + url = make_conninfo(url, host="postgres-restore") + return psycopg.connect(url, autocommit=True, row_factory=dict_row, + options="-c timezone=UTC -c statement_timeout=120000") + + +def snapshot(conn): + counts = {} + for table in ("users", "chats", "chat_members", "messages", "schema_migrations"): + # Имена таблиц — фиксированный список выше, не входные данные клиента. + counts[table] = conn.execute("SELECT count(*) AS n FROM " + table).fetchone()["n"] + digest = hashlib.sha256() + for row in conn.execute("SELECT * FROM messages ORDER BY id"): + digest.update((json.dumps(row, default=str, sort_keys=True, ensure_ascii=False) + "\n").encode()) + samples = [] + offset = 0 + for size in SIZES: + for number in (offset + 1, offset + size // 2, offset + size): + row = conn.execute("SELECT * FROM messages WHERE id=%s", (identifier(f"message:{number}"),)).fetchone() + samples.append(json.loads(json.dumps(row, default=str, sort_keys=True, ensure_ascii=False))) + offset += size + return {"counts": counts, "messages_sha256": digest.hexdigest(), "samples": samples} + + +def benchmark(): + db.pool.open(wait=True, timeout=15) + try: + migrate.apply() + finally: + db.pool.close() + results, plans = [], [] + with connect() as conn: + assert conn.execute("SELECT count(*) AS n FROM users").fetchone()["n"] == 0, "Research requires an empty database" + encoded = passwords.hash_password(secrets.token_urlsafe(24)) + for n in range(1, 11): + conn.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES(%s,%s,%s,%s)", + (identifier(f"user:{n}"), f"research_{n}", "Synthetic research user", encoded)) + for n in range(4): + chat = identifier(f"chat:{n}") + conn.execute("INSERT INTO chats(id,title) VALUES(%s,%s)", (chat, f"Research chat {n}")) + for user in (1, 2): + conn.execute("INSERT INTO chat_members(chat_id,user_id) VALUES(%s,%s)", (chat, identifier(f"user:{user}"))) + variants = [("chat_only", "chat_id"), ("history", "chat_id, created_at, id")] + for repeat in range(3): + order = variants if repeat % 2 == 0 else variants[::-1] + for variant, columns in order: + conn.execute("TRUNCATE messages") + conn.execute("DROP INDEX IF EXISTS messages_history_idx") + conn.execute("DROP INDEX IF EXISTS messages_chat_idx") + index = "messages_history_idx" if variant == "history" else "messages_chat_idx" + conn.execute("CREATE INDEX " + index + " ON messages (" + columns + ")") + lsn = conn.execute("SELECT pg_current_wal_insert_lsn() AS lsn").fetchone()["lsn"] + started = time.perf_counter() + conn.execute("""INSERT INTO messages(id,chat_id,sender_id,text,created_at) + SELECT md5('message:'||n)::uuid, + md5('chat:'||(CASE WHEN n<=70000 THEN 0 WHEN n<=90000 THEN 1 WHEN n<=99000 THEN 2 ELSE 3 END))::uuid, + md5('user:'||(1+n%2))::uuid, + 'Synthetic message '||n||' '||repeat('x',80), + timestamptz '2026-01-01 00:00:00+00' + n*interval '1 microsecond' + FROM generate_series(1,100000) AS n""") + insert_ms = (time.perf_counter() - started) * 1000 + wal = conn.execute("SELECT pg_wal_lsn_diff(pg_current_wal_insert_lsn(), %s) AS bytes", (lsn,)).fetchone()["bytes"] + conn.execute("VACUUM (ANALYZE) messages") + index_bytes = conn.execute("SELECT pg_relation_size(%s::regclass) AS bytes", (index,)).fetchone()["bytes"] + result = {"variant": variant, "repeat": repeat + 1, "insert_100000_ms": insert_ms, + "wal_bytes": int(wal), "index_bytes": index_bytes, "queries": []} + offset = 0 + for chat, size in enumerate(SIZES): + for fraction in (0.1, 0.5, 0.9): + number = offset + int(size * fraction) + params = (identifier(f"chat:{chat}"), START + timedelta(microseconds=number), identifier(f"message:{number}"), 51) + conn.execute(QUERY, params).fetchall() # один прогрев именно этой страницы + times = [] + for _ in range(5): + started = time.perf_counter() + rows = conn.execute(QUERY, params).fetchall() + times.append((time.perf_counter() - started) * 1000) + assert len(rows) == 51 + plan = conn.execute("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + QUERY, params).fetchone()["QUERY PLAN"][0] + result["queries"].append({"chat_size": size, "position": fraction, "latency_ms": times, + "explain_execution_ms": plan["Execution Time"]}) + if repeat == 0: + plans.append({"variant": variant, "chat_size": size, "position": fraction, "plan": plan}) + offset += size + results.append(result) + # Последняя итерация заканчивается production-индексом history. + assert conn.execute("SELECT to_regclass('messages_history_idx') AS name").fetchone()["name"] + baseline = snapshot(conn) + assert baseline["counts"]["messages"] == 100000 + summary = {} + for variant, _ in variants: + trials = [row for row in results if row["variant"] == variant] + latencies = sorted(t for row in trials for q in row["queries"] for t in q["latency_ms"]) + summary[variant] = { + "insert_100000_median_ms": statistics.median(row["insert_100000_ms"] for row in trials), + "index_bytes": trials[-1]["index_bytes"], + "wal_bytes_median": statistics.median(row["wal_bytes"] for row in trials), + "queries": len(latencies), "errors": 0, "concurrency": 1, + "p50_ms": statistics.median(latencies), + "p95_ms": latencies[int(0.95 * (len(latencies) - 1))], + "p99_ms": latencies[int(0.99 * (len(latencies) - 1))], + "successful_sql_rps": len(latencies) / (sum(latencies) / 1000), + } + return {"postgres": conn.execute("SELECT version() AS version").fetchone()["version"], + "messages": 100000, "chat_sizes": SIZES, "repeats": 3, "warmup_per_case": 1, + "measurements_per_case": 5, "page_size": 51, "query": QUERY, + "summary": summary, "trials": results, "plans": plans, "snapshot": baseline} + + +def after_backup(): + with connect() as conn: + message_id = identifier("after-backup") + conn.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES(%s,%s,%s,%s)", + (message_id, identifier("chat:0"), identifier("user:1"), "Committed AFTER backup")) + return {"id": str(message_id), "source_messages": conn.execute("SELECT count(*) AS n FROM messages").fetchone()["n"]} + + +def verify(): + expected = json.load(sys.stdin) + with connect(restored=True) as restored, connect() as source: + actual = snapshot(restored) + assert actual == expected, "Restored data differs from the snapshot" + marker = (identifier("after-backup"),) + assert restored.execute("SELECT 1 FROM messages WHERE id=%s", marker).fetchone() is None + assert source.execute("SELECT 1 FROM messages WHERE id=%s", marker).fetchone() is not None + return {"counts": actual["counts"], "messages_sha256": actual["messages_sha256"], + "samples_verified": len(actual["samples"]), "all_messages_match": True, + "after_backup_message_missing_in_restore": True, + "source_messages": source.execute("SELECT count(*) AS n FROM messages").fetchone()["n"]} + + +if __name__ == "__main__": + if os.environ.get("LAB2_RESEARCH") != "1": + raise SystemExit("Use ./scripts/research.sh: an isolated research database is required") + operation = sys.argv[1] if len(sys.argv) > 1 else "benchmark" + result = {"benchmark": benchmark, "after-backup": after_backup, "verify": verify}[operation]() + print(json.dumps(result, ensure_ascii=False, default=str)) diff --git a/research/observations.py b/research/observations.py new file mode 100644 index 0000000..acc00f6 --- /dev/null +++ b/research/observations.py @@ -0,0 +1,111 @@ +"""Read stored observations through Grafana's provisioned datasource proxy.""" +import json +import os +from pathlib import Path +import subprocess +import time +from urllib.parse import urlencode +from urllib.request import urlopen + +ROOT = Path(__file__).resolve().parents[1] +GRAFANA = os.environ.get('GRAFANA_URL') +if not GRAFANA: + # Also supports an ephemeral host port (GRAFANA_PORT=0) in isolated checks. + address = subprocess.check_output(['docker','compose','port','grafana','3000'], cwd=ROOT, text=True).strip() + GRAFANA = 'http://' + address +GRAFANA = GRAFANA.rstrip('/') + + +def get(path, **params): + url = GRAFANA + path + ('?' + urlencode(params) if params else '') + with urlopen(url, timeout=15) as response: + return json.load(response) + + +def proxy(source, path, **params): + return get('/api/datasources/proxy/uid/' + source + path, **params) + + +def query(expr): + result = proxy('prometheus', '/api/v1/query', query=expr) + assert result['status'] == 'success', result + return result['data']['result'] + + +def scalar(expr, default=0): + result = query(expr) + return float(result[0]['value'][1]) if result else default + + +def logs(expr, start=None, end=None, limit=1000): + return proxy('loki', '/loki/api/v1/query_range', query=expr, + start=str(int((start or time.time()-600)*1e9)), + end=str(int((end or time.time())*1e9)), limit=limit, direction='forward') + + +def trace(trace_id): + return proxy('tempo', '/api/traces/' + trace_id) + + +def compose(*args, check=True, timeout=180): + return subprocess.run(['docker', 'compose', *args], cwd=ROOT, text=True, + capture_output=True, check=check, timeout=timeout) + + +def sql(statement): + return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-c', statement).stdout.strip() + + +def wait(predicate, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + result = predicate() + if result: + return result + except (OSError, ValueError): + pass + time.sleep(1) + raise AssertionError('Timed out waiting for observed condition') + + +def save(name, value): + path = ROOT / 'evidence' / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n') + + +def spans(value): + result = [] + def walk(item, service=''): + if isinstance(item, dict): + if 'resource' in item: + for attr in item['resource'].get('attributes', []): + if attr['key'] == 'service.name': + service = attr['value']['stringValue'] + if 'spanId' in item and 'name' in item: + result.append({'service': service, **item}) + for child in item.values(): + walk(child, service) + elif isinstance(item, list): + for child in item: + walk(child, service) + walk(value) + return result + + +def check_dashboard(): + dashboard = get('/api/dashboards/uid/messenger-lab6')['dashboard'] + checked = [] + for panel in dashboard['panels']: + source = panel.get('datasource', {}).get('uid') + for target in panel.get('targets', []): + if source == 'prometheus': + query(target['expr']) + elif source == 'loki': + assert logs(target['expr'])['status'] == 'success' + elif source == 'tempo': + # Grafana v12 routes traceql frontend searches to this proxy endpoint. + assert proxy('tempo', '/api/search', q=target['query'], limit=20).get('traces') + checked.append({'panel': panel['title'], 'source': source, 'passed': True}) + return {'passed': True, 'panels': checked} diff --git a/research/run.py b/research/run.py new file mode 100644 index 0000000..33e0f0e --- /dev/null +++ b/research/run.py @@ -0,0 +1,99 @@ +"""Создаёт изолированный стенд, измеряет SQL, делает dump и restore; пишет реальные результаты.""" +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +import time +from uuid import uuid4 + + +ROOT = Path(__file__).resolve().parents[1] +project = "lab2-research-" + uuid4().hex[:10] +env = dict(os.environ, COMPOSE_PROJECT_NAME=project) +command = ["docker", "compose", "--profile", "research"] + + +def compose(*args, stdin=None, stdout=subprocess.PIPE): + return subprocess.run([*command, *args], cwd=ROOT, env=env, input=stdin, + stdout=stdout, stderr=subprocess.PIPE, check=True, timeout=600) + + +def run_module(operation, stdin=None): + result = compose("run", "--rm", "--no-deps", "-T", "-e", "LAB2_RESEARCH=1", "api", "python", "-m", "research.benchmark", operation, stdin=stdin) + return json.loads(result.stdout) + + +def wait_database(service): + for _ in range(60): + cid = compose("ps", "-q", service).stdout.decode().strip() + state = subprocess.run(["docker", "inspect", "--format", "{{.State.Health.Status}}", cid], capture_output=True, text=True, check=True) + if state.stdout.strip() == "healthy": + return + time.sleep(1) + raise RuntimeError(service + " did not become healthy") + + +def main(): + evidence = ROOT / "evidence" + evidence.mkdir(exist_ok=True) + backups = ROOT / "backups" + backups.mkdir(exist_ok=True) + dump = backups / (project + ".dump") + metadata = {"utc": datetime.now(timezone.utc).isoformat(), "project": project, + "command": "./scripts/research.sh", "host": platform.system() + " " + platform.machine(), + "commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip() + " + working tree", + "postgres_limits": {"cpus": 1, "memory_mib": 512}, "api_runner_limits": {"cpus": 1, "memory_mib": 256}} + started = time.perf_counter() + try: + print("Research project:", project, flush=True) + compose("up", "-d", "postgres", "postgres-restore") + wait_database("postgres") + wait_database("postgres-restore") + print("Measuring two indexes on the same 100000 messages, three repeats...", flush=True) + result = run_module("benchmark") + plans = result.pop("plans") + (evidence / "query-plans.json").write_text(json.dumps(plans, ensure_ascii=False, indent=2)) + source_id = compose("ps", "-q", "postgres").stdout.decode().strip() + restore_id = compose("ps", "-q", "postgres-restore").stdout.decode().strip() + assert source_id != restore_id + for label, cid in (("source", source_id), ("restored", restore_id)): + inspected = json.loads(subprocess.run(["docker", "inspect", cid], capture_output=True, check=True).stdout)[0] + metadata[label] = {"container": cid, "image": inspected["Image"], + "volumes": [m["Name"] for m in inspected["Mounts"] if m["Type"] == "volume"]} + assert metadata["source"]["volumes"] != metadata["restored"]["volumes"] + metadata["docker"] = subprocess.run(["docker", "info", "--format", "{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}"], capture_output=True, text=True, check=True).stdout.strip() + metadata["stats_after_queries"] = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{json .}}", source_id], capture_output=True, text=True, check=True).stdout.strip() + backup_started = time.perf_counter() + with dump.open("wb") as output: + compose("exec", "-T", "postgres", "pg_dump", "-U", "messenger", "-d", "messenger", "--format=custom", stdout=output) + backup_seconds = time.perf_counter() - backup_started + marker = run_module("after-backup") + restore_started = time.perf_counter() + # Dump подаётся через stdin, без вывода данных или credentials в журнал. + with dump.open("rb") as input_file: + subprocess.run([*command, "exec", "-T", "postgres-restore", "pg_restore", "-U", "messenger", "-d", "messenger", + "--exit-on-error", "--single-transaction", "--no-owner"], cwd=ROOT, env=env, stdin=input_file, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, check=True, timeout=300) + restore_seconds = time.perf_counter() - restore_started + verification_started = time.perf_counter() + verification = run_module("verify", stdin=json.dumps(result["snapshot"]).encode()) + result.update(metadata) + result["backup"] = {"file": "backups/" + dump.name, "bytes": dump.stat().st_size, "seconds": backup_seconds} + result["restore"] = {"seconds": restore_seconds, "verification_seconds": time.perf_counter() - verification_started, + "verification": verification, "post_backup_insert": marker} + result["total_seconds"] = time.perf_counter() - started + (evidence / "research.json").write_text(json.dumps(result, ensure_ascii=False, indent=2)) + print(json.dumps(result["summary"], ensure_ascii=False, indent=2), flush=True) + print(f"PASS: backup {backup_seconds:.2f}s; restore {restore_seconds:.2f}s; 100000 messages, all hashes and 12 samples match", flush=True) + print("PASS: restored into a different container and fresh volume; one post-backup commit is absent as expected", flush=True) + print("Results: evidence/research.json, evidence/query-plans.json; dump stays outside Git in backups/", flush=True) + finally: + compose("down", "--volumes", "--timeout", "10") + print("Removed only temporary research project:", project, flush=True) + + +if __name__ == "__main__": + main() diff --git a/research/slo.py b/research/slo.py new file mode 100644 index 0000000..64a2b4f --- /dev/null +++ b/research/slo.py @@ -0,0 +1,207 @@ +"""Track 2: real-time SLO alert experiment against the current synthetic lab6 stack.""" +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import statistics +import subprocess +import sys +import threading +import time +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(ROOT / 'tests'), str(ROOT)] +from client import Client +from research.observations import compose, logs, proxy, query, save, spans, sql, trace, wait + + +def worker_evidence(requests, phases, started): + stopped = next(p for p in phases if p['name'] == 'worker_stopped') + restored = next(p for p in phases if p['name'] == 'after_worker_recovery') + upload = next(r for r in requests if r['kind'] == 'upload' and r['status'] == 202 + and stopped['active_at'] + 30 < r['utc'] < stopped['end'] - 30) + stored = trace(upload['trace_id']) + chain = spans(stored) + consumers = [s for s in chain if s['name'] == 'attachments process'] + assert consumers, 'Accepted upload must eventually have a real consumer span' + consumed_at = min(int(s['startTimeUnixNano']) / 1e9 for s in consumers) + assert consumed_at >= restored['active_at'] - 2, 'Job was consumed while worker was stopped' + assert {'s3.get_object', 's3.put_object', 'postgresql.UPDATE'} <= {s['name'] for s in chain} + save('lab6-worker-incident-trace.json', stored) + save('lab6-worker-incident-logs.json', logs( + '{service_name=~"messenger-api|messenger-worker"} |= "' + upload['trace_id'] + '"', start=started)) + return {'trace_id': upload['trace_id'], 'request_id': upload['request_id'], + 'accept_to_consume_seconds': round(consumed_at - upload['utc'], 2), + 'consumer_after_restore': True} + + +def main(): + client = Client() + alice, _, _, chat = client.bootstrap() + png = (ROOT / 'tests/fixtures/sample.png').read_bytes() + started = time.time() + stop = threading.Event() + lock = threading.Lock() + requests, samples, phases = [], [], [] + skipped = 0 + phase_name = 'baseline' + pending = set() + + def send(kind): + now = time.time() + name_at_start = phase_name + try: + if kind == 'message': + r = client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', [201,500,503], + actor=alice,body={'text':'synthetic SLO experiment'},headers={'Idempotency-Key':uuid4().hex},schema='Message') + else: + r = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=synthetic.png', [202,500,503], + actor=alice,raw=png,headers={'Content-Type':'image/png'},schema='Attachment') + row = {'utc':now,'phase':name_at_start,'kind':kind,'status':r['status'], + 'duration_seconds':r['elapsed'],'trace_id':r['headers'].get('X-Trace-Id',''), + 'request_id':r['headers']['X-Request-Id']} + except Exception as error: + row = {'utc':now,'phase':name_at_start,'kind':kind,'status':0,'error_type':type(error).__name__, + 'duration_seconds':time.time()-now} + with lock: + requests.append(row) + + def load(): + nonlocal skipped + tick = 0 + next_tick = time.monotonic() + with ThreadPoolExecutor(max_workers=16) as pool: + while not stop.is_set(): + pending.difference_update({f for f in pending if f.done()}) + kinds = ['message'] + (['upload'] if tick % 4 == 0 else []) + for kind in kinds: + if len(pending) >= 16: + skipped += 1 + else: + pending.add(pool.submit(send, kind)) + tick += 1 + next_tick += .5 + stop.wait(max(0,next_tick-time.monotonic())) + + def phase(name, seconds, action=None): + nonlocal phase_name + phase_name = name + row = {'name':name,'start':time.time(),'planned_seconds':seconds} + print('PHASE:', name, seconds, 'seconds', flush=True) + if action: + action() + row['active_at'] = time.time() + deadline = row['start'] + seconds + while time.time() < deadline: + alerts = proxy('prometheus','/api/v1/alerts')['data']['alerts'] + samples.append({'utc':time.time(),'phase':name,'alerts':[ + {'name':a['labels']['alertname'],'state':a['state'],'activeAt':a['activeAt']} for a in alerts], + 'queue_depth':query('max(messenger_queue_depth)'), + 'queue_age':query('max(messenger_queue_oldest_age_seconds)'), + 'api_bad_30s':query('lab6:api_bad_ratio:30s'), + 'api_bad_2m':query('lab6:api_bad_ratio:2m')}) + time.sleep(min(5,max(0,deadline-time.time()))) + row['end'] = time.time() + phases.append(row) + + def db_lock(seconds): + # Named session and bounded pg_sleep: the fault self-releases even if the driver exits. + p = subprocess.Popen(['docker','compose','exec','-T','-e','PGAPPNAME=lab6-incident','postgres', + 'psql','-v','ON_ERROR_STOP=1','-U','messenger','-d','messenger','-c', + f"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({seconds}); COMMIT;"], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + wait(lambda: sql("SELECT count(*) FROM pg_stat_activity WHERE application_name='lab6-incident' AND wait_event='PgSleep'") == '1',timeout=15) + return p + + thread = threading.Thread(target=load) + result = {'utc':datetime.now(timezone.utc).isoformat(),'hypothesis': + 'Multiwindow burn alerts ignore a 4s spike, detect sustained DB/worker degradation and resolve after recovery.', + 'profile':{'message_arrival_per_second':2,'image_arrival_per_second':.5,'max_inflight':16, + 'sampling':1.0,'retries':0,'slo_window_seconds':3600,'api_slo':.99,'ready_slo':.95,'ready_deadline_seconds':30}, + 'docker':subprocess.check_output(['docker','info','--format','{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}'],text=True).strip()} + thread.start() + try: + phase('baseline',120) + phase('short_db_spike',4,lambda:db_lock(4)) + phase('after_short_spike',60) + phase('sustained_db_lock',90,lambda:db_lock(90)) + # Read only saved signals to retain the diagnosis: error trace + correlated log. + failing = next(r for r in reversed(requests) if r['status'] >= 500 and r.get('trace_id')) + time.sleep(3) + saved_trace = trace(failing['trace_id']) + save('lab6-db-incident-trace.json',saved_trace) + save('lab6-db-incident-logs.json',logs('{service_name="messenger-api"} |= "'+failing['trace_id']+'"',start=started)) + result['db_diagnosis'] = {'trace_id':failing['trace_id'],'request_id':failing['request_id'], + 'slow_error_spans':[{'name':s['name'],'status':s.get('status'), + 'duration_ms':(int(s['endTimeUnixNano'])-int(s['startTimeUnixNano']))/1e6, + 'attributes':s.get('attributes',[])} for s in spans(saved_trace) if s.get('status',{}).get('code') in [2,'STATUS_CODE_ERROR']]} + phase('after_db_recovery',100) + phase('worker_stopped',120,lambda:compose('stop','--timeout','5','worker')) + phase('after_worker_recovery',120,lambda:compose('start','worker')) + result['worker_diagnosis'] = worker_evidence(requests, phases, started) + assert all(r['status'] in {201,202,500,503} for r in requests), 'Transport/client errors occurred' + baseline_firing = {a['name'] for s in samples if s['phase']=='baseline' for a in s['alerts'] if a['state']=='firing'} + assert not baseline_firing, baseline_firing + spike_firing = {a['name'] for s in samples if s['phase'] in {'short_db_spike','after_short_spike'} for a in s['alerts'] if a['state']=='firing'} + assert not any('Burn' in a for a in spike_firing), spike_firing + spike_peak = max([float(v['value'][1]) for s in samples + if s['phase'] in {'short_db_spike','after_short_spike'} for v in s['api_bad_30s']] or [0]) + assert spike_peak > 0, 'The real short spike must be visible in the stored error ratio' + result['short_spike_observed_error_ratio_peak'] = spike_peak + firing = {a['name'] for s in samples for a in s['alerts'] if a['state']=='firing'} + expected = {'APIAvailabilityFastBurn','APIAvailabilitySlowBurn','AttachmentReadyFastBurn', + 'AttachmentReadySlowBurn','AttachmentQueueLag','APIHighErrorRate'} + assert expected <= firing, expected-firing + assert not [a for a in samples[-1]['alerts'] if a['state']=='firing'], samples[-1] + result['passed']=True + result['false_positives']={'baseline':len(baseline_firing),'burn_alerts_after_short_spike':0, + 'single_window_alerts_after_short_spike':sorted(spike_firing)} + finally: + stop.set(); thread.join(timeout=20) + # End the bounded lock early on interruption; restore the worker in all cases. + sql("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6-incident'") + compose('start','worker') + ended=time.time() + result.update({'start':started,'end':ended,'duration_seconds':ended-started,'phases':phases, + 'samples':samples,'requests':requests,'skipped_arrivals':skipped}) + summary={} + for p in phases: + rows=[r for r in requests if p['start']<=r['utc']=500 for r in rows), + 'successful_rps':sum(200<=r['status']<300 for r in rows)/(p['end']-p['start']), + **{f'p{q}_ms':round(latencies[min(len(latencies)-1,int(len(latencies)*q/100))],3) + for q in [50,95,99] if latencies}} + result['phase_summary']=summary + # Actual detection and resolution times, measured from observed Prometheus states. + timings={} + for prefix,incident,recovery in [('API','sustained_db_lock','after_db_recovery'),('Attachment','worker_stopped','after_worker_recovery')]: + ip=next((p for p in phases if p['name']==incident),None) + rp=next((p for p in phases if p['name']==recovery),None) + if not ip or not rp: continue + names={a['name'] for s in samples for a in s['alerts'] if a['name'].startswith(prefix) and a['state']=='firing'} + for name in names: + fired=[s['utc'] for s in samples if s['utc']>=ip['start'] and any(a['name']==name and a['state']=='firing' for a in s['alerts'])] + if not fired: continue + resolved=next((s['utc'] for s in samples if s['utc']>=max(rp['active_at'],fired[0]) and not any(a['name']==name and a['state']=='firing' for a in s['alerts'])),None) + timings[name]={'detection_seconds':round(fired[0]-ip['active_at'],2), + 'resolution_after_restore_seconds':round(resolved-rp['active_at'],2) if resolved else None} + result['alert_timings']=timings + expressions={'http_rps':'sum(rate(messenger_http_requests_total[1m]))', + 'http_p95':'histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))', + 'queue_depth':'max(messenger_queue_depth)','queue_age':'max(messenger_queue_oldest_age_seconds)', + 'db_pool':'sum by(state)(messenger_db_pool)','worker_busy':'sum(messenger_worker_busy)', + 'worker_capacity':'sum(messenger_worker_capacity)', + 'cpu':'sum by(service_name)(rate(messenger_process_cpu_time_seconds_total[1m]))', + 'rss':'sum by(service_name)(messenger_process_memory_bytes)', + 'api_bad':'lab6:api_bad_ratio:2m','ready_bad':'lab6:ready_bad_ratio:2m','alerts':'ALERTS{alertstate="firing"}'} + series={name:proxy('prometheus','/api/v1/query_range',query=expr,start=started,end=ended,step=5) for name,expr in expressions.items()} + save('lab6-slo-series.json',series) + save('lab6-slo-research.json',result) + print(json.dumps({k:v for k,v in result.items() if k not in {'requests','samples'}},ensure_ascii=False,indent=2),flush=True) + + +if __name__=='__main__': + main() diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..0a0b999 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +python3 scripts/init_local.py +docker compose build "$@" api diff --git a/scripts/check-docker.sh b/scripts/check-docker.sh new file mode 100755 index 0000000..df5799c --- /dev/null +++ b/scripts/check-docker.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +if [[ ! -x .venv/bin/python ]]; then + echo "Создай .venv и установи requirements.txt по REPORT.md" >&2 + exit 1 +fi +./scripts/build.sh +mkdir -p evidence +.venv/bin/python -u student-tests/lab5_pipeline.py 2>&1 | sed -E 's/[[:blank:]]+$//' | tee evidence/lab5-checks.txt diff --git a/scripts/check-lab6.sh b/scripts/check-lab6.sh new file mode 100755 index 0000000..cfed92c --- /dev/null +++ b/scripts/check-lab6.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +mkdir -p evidence +./scripts/build.sh +# Inherited grade-4 lab5 criteria, on a separate temporary project and fresh volumes. +LAB5_EVIDENCE_FILE=evidence/lab6-lab5-regression.json \ + .venv/bin/python -u student-tests/lab5_pipeline.py 2>&1 | tee evidence/lab6-lab5-regression.txt +./scripts/start.sh +make check validate test PYTHON=.venv/bin/python 2>&1 | tee evidence/lab6-api-checks.txt +.venv/bin/python -m unittest discover -s student-tests -v 2>&1 | tee -a evidence/lab6-api-checks.txt +docker run --rm --entrypoint /bin/promtool -v "$project_dir:/work:ro" -w /work \ + prom/prometheus:v3.4.1 test rules student-tests/promtool-slo.yaml | tee evidence/lab6-promtool.txt +# Only this short check enables the already inherited one-shot worker failpoint. +trap 'ENABLE_FAULT_INJECTION=0 docker compose up -d --no-deps worker >/dev/null 2>&1' EXIT +ENABLE_FAULT_INJECTION=1 docker compose up -d --no-deps worker +.venv/bin/python -u student-tests/lab6_observability.py 2>&1 | tee evidence/lab6-checks.txt +ENABLE_FAULT_INJECTION=0 docker compose up -d --no-deps worker +.venv/bin/python -u student-tests/verify_restart.py 2>&1 | tee evidence/lab6-restart.txt +.venv/bin/python - <<'PY' +import json +from research.observations import ROOT, spans, trace, wait +evidence=json.loads((ROOT/'evidence/lab6-checks.json').read_text()) +trace_id=evidence['checks']['retry_trace_and_logs']['trace_id'] +wait(lambda: len(spans(trace(trace_id))) >= 10) +print('PASS: trace remains queryable after restarting all telemetry stores with their volumes') +PY +.venv/bin/python -u research/slo.py 2>&1 | tee evidence/lab6-slo-research.txt diff --git a/scripts/check.sh b/scripts/check.sh new file mode 100755 index 0000000..90a608d --- /dev/null +++ b/scripts/check.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" + +if [[ ! -x .venv/bin/python ]]; then + echo "Не хватает .venv" >&2 + echo "python3 -m venv .venv" >&2 + echo ".venv/bin/python -m pip install -r requirements.txt" >&2 + exit 1 +fi + +export BASE_URL="${BASE_URL:-http://localhost:8080}" +echo "Проверяем сервер: $BASE_URL" +make check test PYTHON="$project_dir/.venv/bin/python" BASE_URL="$BASE_URL" +.venv/bin/python -m unittest discover -s student-tests -v diff --git a/scripts/demo.sh b/scripts/demo.sh new file mode 100755 index 0000000..af83a80 --- /dev/null +++ b/scripts/demo.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +base_url="${BASE_URL:-http://localhost:8080}" +suffix="$(date +%s)_$RANDOM" +# Только синтетические пароли; токены остаются в памяти shell и не выводятся. +password="Demo-password-$suffix" +register() { + local username="$1" display_name="$2" + jq -nc --arg u "$username" --arg d "$display_name" --arg p "$password" \ + '{username:$u, display_name:$d, password:$p}' | + curl -fsS -H 'Content-Type: application/json' --data-binary @- "$base_url/api/v1/users" | jq -er '.id' +} +login() { + curl -fsS -X POST -u "$1:$password" "$base_url/api/v1/auth/sessions" | jq -er '.token' +} +alice_id="$(register "alice_$suffix" Алиса)" +bob_id="$(register "bob_$suffix" Боб)" +alice_token="$(login "alice_$suffix")" +bob_token="$(login "bob_$suffix")" +chat_id="$(jq -nc --arg a "$alice_id" --arg b "$bob_id" '{title:"Учебный чат",member_ids:[$a,$b]}' | + curl -fsS -H "Authorization: Bearer $alice_token" -H 'Content-Type: application/json' \ + --data-binary @- "$base_url/api/v1/chats" | jq -er '.id')" +for token in "$alice_token" "$bob_token"; do + curl -fsS -H "Authorization: Bearer $token" -H 'Content-Type: application/json' \ + -H "Idempotency-Key: demo-$RANDOM-$RANDOM" \ + -d '{"text":"Привет из lab5! 👋"}' "$base_url/api/v1/chats/$chat_id/messages" | jq . +done +curl -fsS -H "Authorization: Bearer $bob_token" "$base_url/api/v1/chats/$chat_id/messages" | jq . diff --git a/scripts/init_local.py b/scripts/init_local.py new file mode 100644 index 0000000..64b3678 --- /dev/null +++ b/scripts/init_local.py @@ -0,0 +1,62 @@ +"""Создаёт локальное окружение и secrets; существующие значения сохраняет.""" +from pathlib import Path +import json +import os +import re +import secrets +import shutil +from urllib.parse import quote + + +root = Path(__file__).resolve().parents[1] +if not (root / ".env").exists(): + shutil.copyfile(root / ".env.example", root / ".env") +directory = root / "secrets" +directory.mkdir(exist_ok=True, mode=0o700) +directory.chmod(0o700) +password_file = directory / "postgres_password" +if not password_file.exists(): + password_file.write_text(secrets.token_hex(24)) +password = password_file.read_text().strip() +values = { + "database_url": f"postgresql://messenger:{quote(password, safe='')}@postgres:5432/messenger", + "cache_url": "redis://valkey:6379/0", + "broker_password": secrets.token_hex(24), + "s3_root_user": "root-" + secrets.token_hex(8), + "s3_root_password": secrets.token_urlsafe(32), + "s3_access_key_id": "api-" + secrets.token_hex(8), + "s3_secret_access_key": secrets.token_urlsafe(32), +} +for name, value in values.items(): + path = directory / name + if not path.exists(): + path.write_text(value) +broker_url = directory / "broker_url" +if not broker_url.exists(): + broker_url.write_text("redis://:" + (directory / "broker_password").read_text().strip() + "@broker:6379/0") +broker_url.chmod(0o444) +dotenv = dict(line.split("=", 1) for line in (root / ".env").read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") and "=" in line) +bucket = os.environ.get("S3_BUCKET", dotenv.get("S3_BUCKET", "messenger-attachments")) +if re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket) is None: + raise ValueError("Недопустимое имя S3 bucket") +def local_secret(name): + return (directory / name).read_text().strip() +config = {"accounts": [{"id": "messenger-api", "displayName": "messenger-api"}], "identities": [ + {"name": "bootstrap", "credentials": [{"accessKey": local_secret("s3_root_user"), + "secretKey": local_secret("s3_root_password")}], "actions": ["Admin"]}, + {"name": "messenger-api", "account": {"id": "messenger-api", "displayName": "messenger-api"}, + "credentials": [{"accessKey": local_secret("s3_access_key_id"), "secretKey": local_secret("s3_secret_access_key")}], + "actions": ["Read:" + bucket, "List:" + bucket, "Write:" + bucket + "/attachments/*"]}, +]} +config_file = directory / "s3_config" +encoded_config = json.dumps(config) +if not config_file.exists() or config_file.read_text() != encoded_config: + temporary = directory / "s3_config.tmp" + temporary.write_text(encoded_config) + temporary.chmod(0o444) + temporary.replace(config_file) +# Каталог 0700 защищает файлы на хосте; 0444 позволяет USER 10001 читать bind-mounted secret. +for name in ("postgres_password", "s3_config", *values): + (directory / name).chmod(0o444) +print("Local .env and secrets are ready; values are not printed") diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100755 index 0000000..9f0bb21 --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +./scripts/build.sh +docker compose run --rm migrate diff --git a/scripts/replicas.py b/scripts/replicas.py new file mode 100644 index 0000000..e9cfe3e --- /dev/null +++ b/scripts/replicas.py @@ -0,0 +1,90 @@ +"""Управление репликами только текущего Compose project через Docker CLI.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import time +from urllib.request import urlopen + +ROOT = Path(__file__).resolve().parents[1] + + +def command(*args): + return subprocess.check_output(args, cwd=ROOT, text=True, timeout=120).strip() + + +def replicas(): + ids = command("docker", "compose", "ps", "-q", "api").split() + return json.loads(command("docker", "inspect", *ids)) if ids else [] + + +def base_url(): + config = json.loads(command("docker", "compose", "config", "--format", "json")) + port = config["services"]["lb"]["ports"][0]["published"] + return os.environ.get("BASE_URL", "http://127.0.0.1:" + str(port)).rstrip("/") + + +def wait_ready(): + url = base_url() + "/health/ready" + deadline = time.monotonic() + 90 + seen = set() + while time.monotonic() < deadline: + current = replicas() + expected = {info["Config"]["Hostname"] for info in current} + if expected and all(info["State"].get("Health", {}).get("Status") == "healthy" for info in current): + try: + with urlopen(url, timeout=2) as response: + if response.status == 200: + seen.add(response.headers.get("X-Instance-Id")) + if expected <= seen: + print("API ready through LB:", ", ".join(sorted(expected)), flush=True) + return + except OSError: + pass + time.sleep(0.2) + raise RuntimeError("Не все работающие API-реплики готовы через LB за 90 секунд") + + +def stop_one(): + target = os.environ.get("TARGET_INSTANCE_ID") + if not target: + raise RuntimeError("Задайте TARGET_INSTANCE_ID из X-Instance-Id") + current = replicas() + matches = [info for info in current if info["Config"]["Hostname"] == target] + if len(current) < 2 or len(matches) != 1: + raise RuntimeError("Нужны минимум две работающие реплики и ровно одна выбранная") + selected = matches[0] + command("docker", "stop", "--time", "8", selected["Id"]) + print("Stopped API:", target, "container=" + selected["Id"], flush=True) + + +def rolling(): + wait_ready() + originals = replicas() + count = len(originals) + if count < 2: + raise RuntimeError("Для поочерёдной замены нужны минимум две API-реплики") + for old in originals: + active = replicas() + healthy = [info for info in active if info["Id"] != old["Id"] + and info["State"].get("Health", {}).get("Status") == "healthy"] + if not healthy: + raise RuntimeError("Нет готовой реплики, которая продолжит обслуживать запросы") + print("Replacing API:", old["Config"]["Hostname"], flush=True) + command("docker", "stop", "--time", "8", old["Id"]) + command("docker", "rm", old["Id"]) + # --no-recreate сохраняет остальные процессы; Compose создаёт только недостающий. + command("docker", "compose", "up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={count}", "api") + wait_ready() + assert len(replicas()) == count + final = replicas() + assert not ({info["Id"] for info in originals} & {info["Id"] for info in final}) + print("PASS: all API containers replaced sequentially; stores and LB kept running", flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["wait", "stop", "rolling"]) + action = parser.parse_args().action + {"wait": wait_ready, "stop": stop_one, "rolling": rolling}[action]() diff --git a/scripts/research-lab2.sh b/scripts/research-lab2.sh new file mode 100755 index 0000000..b4f8e34 --- /dev/null +++ b/scripts/research-lab2.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +./scripts/build.sh +mkdir -p evidence +python3 -u research/run.py 2>&1 | tee evidence/research.txt diff --git a/scripts/research.sh b/scripts/research.sh new file mode 100755 index 0000000..8b869a7 --- /dev/null +++ b/scripts/research.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +python3 scripts/wait-observability.py +mkdir -p evidence +.venv/bin/python -u research/slo.py 2>&1 | tee evidence/lab6-slo-research.txt diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..517562a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +docker compose restart --timeout 8 postgres valkey s3-storage s3 broker api lb cleanup publisher worker otel-collector prometheus loki tempo grafana +./scripts/wait-ready.sh +python3 scripts/wait-observability.py diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..b5b225f --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +# Сначала собираем образ и применяем только обратно совместимые миграции. +./scripts/migrate.sh +exec python3 scripts/replicas.py rolling diff --git a/scripts/run-local.sh b/scripts/run-local.sh new file mode 100755 index 0000000..1c0b0e7 --- /dev/null +++ b/scripts/run-local.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +# Локальный стенд: HAProxy, две API-реплики, PostgreSQL и Valkey. +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +exec "$project_dir/scripts/start.sh" diff --git a/scripts/start-worker.sh b/scripts/start-worker.sh new file mode 100755 index 0000000..15b0c61 --- /dev/null +++ b/scripts/start-worker.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +docker compose start worker diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100755 index 0000000..81a6c27 --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +python3 scripts/init_local.py +docker compose up -d --build lb cleanup publisher worker otel-collector prometheus grafana +./scripts/wait-ready.sh +python3 scripts/wait-observability.py diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..b77ec3f --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +exec python3 scripts/replicas.py stop diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100755 index 0000000..5a0ac7a --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +docker compose stop --timeout 8 worker diff --git a/scripts/stop.sh b/scripts/stop.sh new file mode 100755 index 0000000..7dfffac --- /dev/null +++ b/scripts/stop.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +docker compose down --timeout 10 +echo "Контейнеры удалены; именованные volumes сохранены" diff --git a/scripts/wait-observability.py b/scripts/wait-observability.py new file mode 100644 index 0000000..68d50d1 --- /dev/null +++ b/scripts/wait-observability.py @@ -0,0 +1,26 @@ +"""Readiness of provisioned telemetry, queried through the local Grafana proxy.""" +from pathlib import Path +import sys +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from research.observations import get, proxy, query, wait + + +def ready(): + assert get('/api/health')['database'] == 'ok' + assert get('/api/dashboards/uid/messenger-lab6')['dashboard']['uid'] == 'messenger-lab6' + assert len(get('/api/datasources')) == 3 + assert len(query('up == 1')) == 3 + proxy('loki', '/loki/api/v1/labels') + proxy('tempo', '/api/search', limit=1) + return True + + +if __name__ == '__main__': + # Assertion failures are readiness failures, not successful probes. + def probe(): + try: + return ready() + except (AssertionError, OSError, ValueError): + return False + wait(probe, timeout=90) + print('PASS: Collector scrape, stores, Grafana datasources and dashboard are ready') diff --git a/scripts/wait-ready.sh b/scripts/wait-ready.sh new file mode 100755 index 0000000..da08577 --- /dev/null +++ b/scripts/wait-ready.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$project_dir" +exec python3 scripts/replicas.py wait diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 0000000..420cbd4 --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,16 @@ +"""Общий запуск""" + +import os + +import uvicorn + + +if __name__ == "__main__": + uvicorn.run( + "src.app:app", + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=int(os.environ.get("APP_PORT", "8080")), + workers=1, + access_log=False, + timeout_graceful_shutdown=5, + ) diff --git a/src/app.py b/src/app.py new file mode 100644 index 0000000..072b489 --- /dev/null +++ b/src/app.py @@ -0,0 +1,222 @@ +"""HTTP-адреса, определение пользователя и обработка запросов""" + +from contextlib import asynccontextmanager +import json +import logging +import re +from time import perf_counter +from uuid import UUID, uuid4 +from urllib.parse import quote + +from fastapi import FastAPI, Request, Response +from fastapi.responses import StreamingResponse +from starlette.concurrency import run_in_threadpool +from fastapi.exceptions import RequestValidationError +from starlette.exceptions import HTTPException + +from src import attachments, auth, db, migrate, object_store, passwords, storage, telemetry +from src.errors import ApiError, error_response +from src.models import CreateChat, CreateMessage, CreateUser +from src.config import INSTANCE_ID + + +@asynccontextmanager +async def lifespan(app): + telemetry.setup('messenger-api', db.pool) + # Expose zeros before the first error. Otherwise rate() cannot see a short + # burst that finishes before the first scrape of a new status-class series. + for route in app.routes: + if getattr(route, 'path', '').startswith('/api/v1/'): + for method in route.methods: + for status_class in ('2xx', '3xx', '4xx', '5xx'): + telemetry.http_requests.add(0, {'method': method, 'route': route.path, + 'status_class': status_class}) + await run_in_threadpool(db.pool.open, wait=True, timeout=15) + try: + await run_in_threadpool(migrate.check) + yield + finally: + await run_in_threadpool(db.pool.close) + await run_in_threadpool(telemetry.shutdown) + + +app = FastAPI(title="Учебный мессенджер", redirect_slashes=False, lifespan=lifespan, + telemetry={'auto_configure': False, 'tracing': False, 'metrics': False, + 'logs': False, 'operation_spans': False}) +logger = logging.getLogger("uvicorn.error") + + +@app.middleware("http") +async def handle_request(request: Request, call_next): + path = request.url.path + route = next((route.path for route in app.routes + if hasattr(route, 'path_regex') and route.path_regex.fullmatch(path)), 'unmatched') + method = request.method if request.method in {'GET', 'POST', 'DELETE', 'PUT', 'PATCH', 'HEAD', 'OPTIONS'} else 'OTHER' + attrs = {'method': method, 'route': route} + started = perf_counter() + if path.startswith('/health/') or not path.startswith('/api/v1/'): + return await process_request(request, call_next) + with telemetry.span(method + ' ' + route, kind=telemetry.SpanKind.SERVER, parent=request.headers, + attributes={'http.request.method': method, 'http.route': route}) as current: + response = await process_request(request, call_next) + current.set_attribute('http.response.status_code', response.status_code) + if response.status_code >= 500: + current.set_status(telemetry.Status(telemetry.StatusCode.ERROR)) + context = current.get_span_context() + if context.is_valid: + response.headers['X-Trace-Id'] = format(context.trace_id, '032x') + elapsed = perf_counter() - started + attrs['status_class'] = str(response.status_code // 100) + 'xx' + telemetry.http_requests.add(1, attrs) + telemetry.http_duration.record(elapsed, attrs) + telemetry.event('http.request', 'ERROR' if response.status_code >= 500 else 'INFO', + method=method, route=route, status=response.status_code, duration_seconds=round(elapsed, 6), + request_id=request.state.request_id) + return response + + +async def process_request(request: Request, call_next): + request.state.request_id = str(uuid4()) + started = perf_counter() + path = request.url.path + try: + is_registration = request.method == "POST" and path == "/api/v1/users" + is_login = request.method == "POST" and path == "/api/v1/auth/sessions" + is_upload = request.method == "POST" and re.fullmatch(r"/api/v1/chats/[^/]+/attachments", path) + if is_login: + request.state.user = await run_in_threadpool(auth.basic_user, request.headers.get("Authorization")) + if await request.body(): + raise ApiError(400, "invalid_request", "Вход через Basic не принимает тело запроса") + elif path.startswith("/api/v1/") and not is_registration: + request.state.user = await run_in_threadpool(auth.current_user, request.headers.get("Authorization")) + if request.method == "POST" and path.startswith("/api/v1/") and not is_login and not is_upload: + content_type = request.headers.get("Content-Type", "").split(";", 1)[0].strip().lower() + if content_type != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + try: + json.loads((await request.body()).decode("utf-8")) + except (ValueError, RecursionError): + raise ApiError(400, "invalid_request", "Ожидается корректный JSON в UTF-8") + response = await call_next(request) + except ApiError as error: + response = error_response(request, error) + except Exception as error: + # Не печатаем тело запроса или текст исключения: там могут быть сообщения. + telemetry.event('http.exception', 'ERROR', error_type=type(error).__name__, request_id=request.state.request_id) + response = error_response(request, ApiError(500, "internal_error", "Внутренняя ошибка сервера")) + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +def handle_api_error(request: Request, error: ApiError): + return error_response(request, error) + + +@app.exception_handler(RequestValidationError) +def handle_validation_error(request: Request, error: RequestValidationError): + # Контракт требует 400 и наш формат ошибки, а не стандартный ответ FastAPI. + return error_response(request, ApiError(400, "invalid_request", "Неверные поля или параметры запроса")) + + +@app.exception_handler(HTTPException) +def handle_http_error(request: Request, error: HTTPException): + code = "not_found" if error.status_code == 404 else "invalid_request" + message = "Адрес не найден" if error.status_code == 404 else "Недопустимый HTTP-запрос" + response = error_response(request, ApiError(error.status_code, code, message)) + if error.headers: + response.headers.update(error.headers) + return response + + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + db.ready() + auth.ready() + object_store.ready() + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +def create_user(data: CreateUser): + return storage.create_user(data.username, data.display_name, passwords.hash_password(data.password)) + + +@app.get("/api/v1/users/me") +def current_user(request: Request): + return request.state.user + + +@app.post("/api/v1/chats", status_code=201) +def create_chat(request: Request, data: CreateChat): + return storage.create_chat(request.state.user["id"], data.title, data.member_ids) + + +@app.get("/api/v1/chats") +def list_chats(request: Request, limit: str = "50", cursor: str | None = None): + user_id = request.state.user["id"] + return storage.list_chats(user_id, limit, cursor) + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(request: Request, chat_id: UUID): + return storage.get_chat(request.state.user["id"], str(chat_id)) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message(request: Request, chat_id: UUID, data: CreateMessage): + keys = request.headers.getlist("Idempotency-Key") + if len(keys) != 1 or re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", keys[0]) is None: + raise ApiError(400, "invalid_request", "Нужен Idempotency-Key: 1–128 символов ASCII [A-Za-z0-9._:-]") + return storage.create_message(request.state.user["id"], str(chat_id), data.text, keys[0], data.attachment_ids) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(request: Request, chat_id: UUID, filename: str): + return await attachments.upload(request, str(chat_id), filename) + + +@app.get("/api/v1/attachments/{attachment_id}") +def get_attachment(request: Request, attachment_id: UUID): + return attachments.public_attachment(attachments.find(request.state.user["id"], str(attachment_id))) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment(request: Request, attachment_id: UUID, variant: str = "original"): + row = attachments.find(request.state.user["id"], str(attachment_id)) + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение не готово") + info = row + key = row["object_key"] + if variant != "original": + info = next((item for item in row["variants"] if item["name"] == variant), None) + if info is None: + raise ApiError(404, "not_found", "Вариант вложения не найден") + key = row["object_key"].rsplit("/", 1)[0] + "/" + variant + response = object_store.get(key) + return StreamingResponse(attachments.chunks(response), media_type=info["content_type"], headers={ + "Content-Length": str(info["size_bytes"]), "X-Content-Type-Options": "nosniff", + "Content-Disposition": "attachment; filename*=UTF-8''" + quote(row["filename"], safe=""), + }) + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(request: Request, chat_id: UUID, limit: str = "50", cursor: str | None = None): + return storage.list_messages(request.state.user["id"], str(chat_id), limit, cursor) + + +@app.post("/api/v1/auth/sessions", status_code=201) +def create_session(request: Request): + return auth.create_session(request.state.user) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_session(request: Request): + auth.logout(request.headers.get("Authorization")) + return Response(status_code=204) diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 0000000..aa2fee0 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,154 @@ +"""Приём и проверка -> durable S3 staging + PostgreSQL outbox -> 202.""" +import asyncio +import hashlib +import os +from tempfile import TemporaryFile +import unicodedata +from uuid import UUID, uuid4 +import warnings + +from PIL import Image, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect + +from src import db, jobs, object_store, storage, telemetry +from src.errors import ApiError + + +MAX_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) +MAX_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) +if not 1 <= MAX_BYTES <= 10485760 or not 1 <= MAX_PIXELS <= 40000000: + raise ValueError("Лимиты загрузки должны быть положительными и не превышать контракт") +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +warnings.filterwarnings("error", category=Image.DecompressionBombWarning) +# Максимум одно полное декодирование на реплику, независимо от числа клиентов. +upload_slot = asyncio.Semaphore(1) + + +def public_attachment(row): + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), + "filename": row["filename"], "content_type": row["content_type"], + "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"], + "variants": row["variants"], "error_code": row["error_code"], "created_at": storage.utc(row["created_at"]), + "updated_at": storage.utc(row["updated_at"])} + + +def validate_filename(filename): + if not 1 <= len(filename) <= 255 or not filename.strip() or any( + char in "/\\" or unicodedata.category(char) == "Cc" for char in filename + ): + raise ApiError(400, "invalid_request", "Недопустимое имя файла") + try: + filename.encode("utf-8") + except UnicodeError: + raise ApiError(400, "invalid_request", "Недопустимое имя файла") from None + + +def validate_image(file, declared_type): + try: + file.seek(0) + with Image.open(file, formats=["PNG", "JPEG"]) as image: + true_type = {"PNG": "image/png", "JPEG": "image/jpeg"}[image.format] + if image.width * image.height > MAX_PIXELS: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + if true_type != declared_type: + raise ApiError(415, "unsupported_media_type", "MIME не совпадает с содержимым") + image.verify() + # verify проверяет структуру/CRC, load дополнительно обнаруживает обрыв pixel data. + file.seek(0) + with Image.open(file, formats=["PNG", "JPEG"]) as image: + image.load() + return true_type + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise ApiError(413, "payload_too_large", "Слишком много пикселей") from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise ApiError(415, "unsupported_media_type", "Повреждённый PNG/JPEG или другой формат") from None + + +def lock_upload(conn, key, blocking=True): + function = "pg_advisory_xact_lock" if blocking else "pg_try_advisory_xact_lock" + return conn.execute(f"SELECT {function}(hashtextextended(%s, 4)) AS locked", (key,)).fetchone()["locked"] + + +def save(user_id, chat_id, filename, content_type, file, size, sha256): + attachment_id, job_id = uuid4(), uuid4() + staging_key = "attachments/staging/" + attachment_id.hex + final_key = "attachments/" + attachment_id.hex + "/v1/original" + # Журнал фиксируется до PUT: crash до записи job оставит управляемый orphan. + with db.connection() as conn: + storage.require_member(conn, user_id, chat_id) + conn.execute("INSERT INTO attachment_uploads(id, object_key) VALUES (%s, %s)", (attachment_id, staging_key)) + with db.connection() as conn: + conn.execute("SET LOCAL idle_in_transaction_session_timeout='30s'") + lock_upload(conn, staging_key) + if not conn.execute("SELECT 1 FROM attachment_uploads WHERE id=%s", (attachment_id,)).fetchone(): + raise ApiError(503, "unavailable", "Загрузка больше не активна") + object_store.put(staging_key, file, size, content_type) + row = conn.execute("""INSERT INTO attachments(id,chat_id,owner_id,object_key,filename, + content_type,size_bytes,sha256,status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *""", + (attachment_id, UUID(chat_id), UUID(user_id), final_key, filename, content_type, size, sha256)).fetchone() + context = telemetry.carrier() + conn.execute("""INSERT INTO attachment_jobs(id,attachment_id,staging_key,max_attempts,traceparent,tracestate) + VALUES (%s,%s,%s,%s,%s,%s)""", (job_id, attachment_id, staging_key, jobs.MAX_ATTEMPTS, + context.get('traceparent'), context.get('tracestate'))) + conn.execute("INSERT INTO attachment_outbox(job_id,attempt) VALUES (%s,1)", (job_id,)) + conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (attachment_id,)) + result = public_attachment(row) + # Снимок queued даже если worker успеет закончить до отправки ответа. + telemetry.event('attachment.accepted', job_id=str(job_id), attachment_id=str(attachment_id)) + return result + + +async def upload(request, chat_id, filename): + user_id = request.state.user["id"] + await run_in_threadpool(storage.get_chat, user_id, chat_id) + validate_filename(filename) + content_type = request.headers.get("Content-Type", "").split(";", 1)[0].strip().lower() + if content_type not in {"image/png", "image/jpeg"}: + raise ApiError(415, "unsupported_media_type", "Ожидаются сырые bytes PNG/JPEG") + length = request.headers.get("Content-Length") + if length is not None: + if not length.isascii() or not length.isdecimal(): + raise ApiError(400, "invalid_request", "Неверный Content-Length") + if len(length) > 10 or int(length) > MAX_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше допустимого размера") + try: + await asyncio.wait_for(upload_slot.acquire(), timeout=20) + except TimeoutError: + raise ApiError(503, "unavailable", "Загрузки заняты; повторите позднее") from None + try: + with TemporaryFile() as file: + size = 0 + digest = hashlib.sha256() + try: + async with asyncio.timeout(30): + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше допустимого размера") + digest.update(chunk) + await run_in_threadpool(file.write, chunk) + except (TimeoutError, ClientDisconnect): + raise ApiError(400, "invalid_request", "Загрузка прервана или истёк таймаут") from None + true_type = await run_in_threadpool(validate_image, file, content_type) + return await run_in_threadpool(save, user_id, chat_id, filename, true_type, file, size, digest.hexdigest()) + finally: + upload_slot.release() + + +def find(user_id, attachment_id): + with db.connection() as conn: + row = conn.execute("""SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id + WHERE a.id=%s AND m.user_id=%s""", (UUID(attachment_id), UUID(user_id))).fetchone() + if row is None: + raise ApiError(404, "not_found", "Вложение не найдено") + return row + + +def chunks(response): + try: + while chunk := response.read(65536): + yield chunk + finally: + response.close() + response.release_conn() diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 0000000..a8db643 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,95 @@ +"""Basic только для входа; Bearer-сессии с TTL и AOF находятся в Valkey.""" +import base64 +import binascii +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +import hashlib +import json +import secrets + +import redis + +from src import passwords, storage +from src.config import SESSION_TTL, secret +from src.errors import ApiError + + +sessions = redis.Redis.from_url( + secret("CACHE_URL"), decode_responses=True, + socket_connect_timeout=0.5, socket_timeout=0.5, max_connections=10, + retry=redis.retry.Retry(redis.backoff.NoBackoff(), 0), +) + + +@contextmanager +def available(): + try: + yield + except redis.RedisError: + raise ApiError(503, "unavailable", "Valkey временно недоступен") from None + + +def unauthorized(basic=False): + challenge = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' + return ApiError(401, "unauthorized", "Неверная или отсутствующая идентификация", {"WWW-Authenticate": challenge}) + + +def credential(header, scheme): + parts = (header or "").split() + if len(parts) != 2 or parts[0].lower() != scheme.lower(): + raise unauthorized(scheme == "Basic") + return parts[1] + + +def basic_user(header): + value = credential(header, "Basic") + try: + decoded = base64.b64decode(value, validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, UnicodeError, binascii.Error): + raise unauthorized(True) from None + user = storage.find_login(username) + if user is None or not passwords.matches(user["password_hash"], password): + raise unauthorized(True) + return storage.public_user(user) + + +def session_key(token): + return "session:" + hashlib.sha256(token.encode()).hexdigest() + + +def create_session(user): + token = secrets.token_urlsafe(32) + expires = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL) + value = {"user_id": user["id"], "expires_at": storage.utc(expires)} + with available(): + sessions.set(session_key(token), json.dumps(value), ex=SESSION_TTL) + return {"token": token, "token_type": "Bearer", "expires_at": value["expires_at"], "user": user} + + +def current_user(header): + token = credential(header, "Bearer") + with available(): + value = sessions.get(session_key(token)) + if value is None: + raise unauthorized() + data = json.loads(value) + if datetime.fromisoformat(data["expires_at"].replace("Z", "+00:00")) <= datetime.now(timezone.utc): + raise unauthorized() + user = storage.find_user(data["user_id"]) + if user is None: + raise unauthorized() + return user + + +def logout(header): + token = credential(header, "Bearer") + with available(): + removed = sessions.delete(session_key(token)) + if not removed: + raise unauthorized() + + +def ready(): + with available(): + sessions.ping() diff --git a/src/bootstrap_s3.py b/src/bootstrap_s3.py new file mode 100644 index 0000000..254c6fe --- /dev/null +++ b/src/bootstrap_s3.py @@ -0,0 +1,38 @@ +"""Идемпотентное создание приватного bucket отдельной административной учёткой.""" +import os +import time +from urllib.parse import urlsplit + +from minio import Minio +from minio.error import MinioException, S3Error +import urllib3 + +from src.config import secret + + +def bootstrap(): + endpoint = urlsplit(os.environ.get("S3_ENDPOINT", "http://s3:9000")) + client = Minio(endpoint.netloc, access_key=secret("S3_ROOT_USER"), secret_key=secret("S3_ROOT_PASSWORD"), + secure=endpoint.scheme == "https", region=os.environ.get("S3_REGION", "us-east-1"), + http_client=urllib3.PoolManager(timeout=urllib3.Timeout(connect=1, read=3), retries=False)) + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + deadline = time.monotonic() + 60 + while True: + try: + if not client.bucket_exists(bucket): + client.make_bucket(bucket) + print("Private bucket is ready; static IAM permits only authenticated accounts", flush=True) + return + except S3Error as error: + if error.code in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}: + return + if time.monotonic() >= deadline: + raise RuntimeError("S3 bootstrap failed") from None + except (MinioException, urllib3.exceptions.HTTPError, OSError): + if time.monotonic() >= deadline: + raise RuntimeError("S3 bootstrap timed out") from None + time.sleep(1) + + +if __name__ == "__main__": + bootstrap() diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 0000000..2c259a0 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,46 @@ +"""Один durable Stream, consumer group; ACK только после эффекта/устойчивого retry.""" +import json + +import redis + +from src.config import secret +from src import telemetry + +STREAM = 'attachments:v1' +GROUP = 'image-workers' + + +def connect(): + return redis.Redis.from_url(secret('BROKER_URL'), decode_responses=True, + socket_connect_timeout=2, socket_timeout=3, + retry_on_timeout=False) + + +def bootstrap(client): + try: + client.xgroup_create(STREAM, GROUP, id='0', mkstream=True) + except redis.ResponseError as error: + if 'BUSYGROUP' not in str(error): + raise + + +def publish(client, job): + payload = {'schema_version': 1, 'job_id': str(job['id']), 'type': 'attachment.process.v1', + 'attachment_id': str(job['attachment_id']), 'pipeline_version': job['pipeline_version'], + 'attempt': job['attempt'], 'created_at': job['created_at'].isoformat().replace('+00:00', 'Z')} + with telemetry.span('attachments publish', kind=telemetry.SpanKind.PRODUCER, + parent={key: job[key] for key in ('traceparent', 'tracestate') if job.get(key)}, + attributes={'messaging.system': 'redis', 'messaging.destination.name': STREAM, + 'messaging.operation.type': 'send', 'job.attempt': job['attempt']}): + payload.update(telemetry.carrier()) + result = client.xadd(STREAM, {'job': json.dumps(payload)}) + telemetry.event('job.published', job_id=str(job['id']), attempt=job['attempt']) + return result + + +def acknowledge(client, message_id): + # Единственная группа: подтверждённые записи можно удалить. Pending не trim-им. + with client.pipeline(transaction=True) as pipe: + pipe.xack(STREAM, GROUP, message_id) + pipe.xdel(STREAM, message_id) + pipe.execute() diff --git a/src/cleanup.py b/src/cleanup.py new file mode 100644 index 0000000..0a69808 --- /dev/null +++ b/src/cleanup.py @@ -0,0 +1,87 @@ +"""Очистка orphan PUT и staging только ready/просроченных failed; текущую работу не трогает.""" +import argparse +import json +import logging +import os +import time + +from src import db, object_store +from src.attachments import lock_upload + + +GRACE_SECONDS = int(os.environ.get("UPLOAD_CLEANUP_GRACE_SECONDS", "3600")) +if GRACE_SECONDS < 60: + raise ValueError("Безопасный grace period должен быть не меньше 60 секунд") + + +def cleanup_orphans(): + result = {"removed": 0, "kept_ready": 0, "skipped_active": 0} + with db.connection() as conn: + candidates = conn.execute("""SELECT id, object_key FROM attachment_uploads + WHERE created_at < clock_timestamp() - %s * interval '1 second' + ORDER BY created_at LIMIT 100""", (GRACE_SECONDS,)).fetchall() + for row in candidates: + with db.connection() as conn: + if not lock_upload(conn, row["object_key"], blocking=False): + result["skipped_active"] += 1 + continue + pending = conn.execute("""SELECT 1 FROM attachment_uploads WHERE id=%s + AND created_at < clock_timestamp() - %s * interval '1 second' FOR UPDATE""", + (row["id"], GRACE_SECONDS)).fetchone() + if not pending: + continue + ready = conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (row["object_key"],)).fetchone() + if ready: + result["kept_ready"] += 1 + else: + object_store.remove(row["object_key"]) + result["removed"] += 1 + conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (row["id"],)) + return result + + +RETENTION = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400")) +if RETENTION < 60: + raise ValueError("STAGING_RETENTION_SECONDS должен быть >=60") + + +def cleanup_staging(): + removed = 0 + with db.connection() as conn: + rows = conn.execute("""SELECT j.*,a.status,a.object_key FROM attachment_jobs j + JOIN attachments a ON a.id=j.attachment_id WHERE j.staging_deleted_at IS NULL + AND (a.status='ready' OR (a.status='failed' AND a.updated_at Image.MAX_IMAGE_PIXELS: + raise ValueError('pixel_limit') + image.load() + ImageOps.exif_transpose(image, in_place=True) + if image.mode in ('RGBA', 'LA') or 'transparency' in image.info: + rgba = image.convert('RGBA') + rgb = Image.new('RGB', image.size, 'white') + rgb.paste(rgba, mask=rgba.getchannel('A')) + rgba.close() + else: + rgb = image.convert('RGB') + rgb.info.clear() + variants = [] + for name, size, encoding, mime in [ + ('thumbnail', fit_size(rgb.size, 256), 'JPEG', 'image/jpeg'), + ('crop', (128, 128), 'JPEG', 'image/jpeg'), + ('optimized', fit_size(rgb.size, 1280), 'WEBP', 'image/webp'), + ]: + variant = ImageOps.fit(rgb, size, method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) if name == 'crop' else rgb.resize(size, Image.Resampling.LANCZOS) + variant.info.clear() + buffer = io.BytesIO() + variant.save(buffer, format=encoding, quality=85 if encoding == 'JPEG' else 80, + **({'method': 4} if encoding == 'WEBP' else {})) + data = buffer.getvalue() + (output / name).write_bytes(data) + variants.append({'name': name, 'content_type': mime, 'size_bytes': len(data), + 'width': size[0], 'height': size[1], 'sha256': hashlib.sha256(data).hexdigest()}) + variant.close() + rgb.close() + (output / 'variants.json').write_text(json.dumps(variants)) + + +if __name__ == '__main__': + memory = int(os.environ.get('WORKER_TASK_MEMORY_MB', '640')) * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + seconds = int(os.environ.get('JOB_TIMEOUT_SECONDS', '60')) + resource.setrlimit(resource.RLIMIT_CPU, (seconds, seconds + 1)) + try: + from src import faults + faults.hit(sys.argv[3], 'transform') + transform(Path(sys.argv[1]), Path(sys.argv[2])) + except (ValueError, OSError, SyntaxError, Image.DecompressionBombWarning, Image.DecompressionBombError): + sys.exit(65) # permanent invalid_image; stderr не содержит внутренние данные. + except MemoryError: + sys.exit(66) diff --git a/src/job_admin.py b/src/job_admin.py new file mode 100644 index 0000000..842301b --- /dev/null +++ b/src/job_admin.py @@ -0,0 +1,29 @@ +"""Локальная операторская CLI: failed registry и безопасный replay по job_id.""" +import argparse +import json + +from src import db, jobs + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + sub.add_parser('failed') + replay = sub.add_parser('replay') + replay.add_argument('job_id') + args = parser.parse_args() + db.pool.open(wait=True, timeout=15) + try: + if args.command == 'replay': + jobs.replay(args.job_id) + print('Replay queued with audit; job_id=' + args.job_id) + else: + with db.connection() as conn: + rows = conn.execute("""SELECT j.id AS job_id,j.attachment_id,j.attempt,j.max_attempts, + a.error_code,j.last_error,j.staging_deleted_at FROM attachment_jobs j + JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' ORDER BY a.updated_at""").fetchall() + print(json.dumps(rows, default=str, indent=2)) + except ValueError as error: + parser.exit(2, str(error) + '\n') + finally: + db.pool.close() diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 0000000..2a64e86 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,103 @@ +"""Транзакционные переходы состояний, bounded retry и явный audit replay.""" +import os +import random +from uuid import UUID, uuid4 + +from src import db, telemetry + +MAX_ATTEMPTS = int(os.environ.get('JOB_MAX_ATTEMPTS', '5')) +TIMEOUT = int(os.environ.get('JOB_TIMEOUT_SECONDS', '60')) +LEASE = int(os.environ.get('JOB_LEASE_SECONDS', '90')) +BACKOFF = float(os.environ.get('JOB_BACKOFF_SECONDS', '2')) +if not 1 <= MAX_ATTEMPTS <= 20 or TIMEOUT < 1 or LEASE < TIMEOUT + 5 or BACKOFF <= 0: + raise ValueError('Неверные retry/timeout/lease настройки') + + +def event(conn, job, name, error=None): + conn.execute('INSERT INTO attachment_job_events(job_id,attempt,event,error_code) VALUES (%s,%s,%s,%s)', + (job['id'], job['attempt'], name, error)) + + +def locked(conn, job_id): + return conn.execute('''SELECT j.*, a.status, a.object_key, a.variants, a.sha256, a.size_bytes, + a.content_type FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id + WHERE j.id=%s FOR UPDATE OF j,a''', (job_id,)).fetchone() + + +def claim(payload): + with db.connection() as conn: + job = locked(conn, UUID(payload['job_id'])) + if not job or str(job['attachment_id']) != payload['attachment_id'] or job['pipeline_version'] != payload['pipeline_version']: + return 'obsolete', None + if job['status'] in {'ready', 'failed'}: + return job['status'], job + if payload['attempt'] != job['attempt']: + return 'obsolete', None + if job['status'] != 'queued': + return 'busy', None + row = conn.execute('''UPDATE attachment_jobs SET lease_token=%s, + lease_until=clock_timestamp()+%s*interval '1 second' WHERE id=%s + AND next_attempt_at<=clock_timestamp() RETURNING *''', (uuid4(), LEASE, job['id'])).fetchone() + if not row: + return 'busy', None + job.update(row) + conn.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s", (job['attachment_id'],)) + event(conn, job, 'processing') + return 'claimed', job + + +def retry_or_fail(conn, job, code, permanent=False): + # replay_count задаёт новый ограниченный бюджет, сохраняя глобальный номер attempt. + exhausted = job['attempt'] >= job['max_attempts'] + if permanent or exhausted: + final_code = code if permanent else 'retry_exhausted' + conn.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=clock_timestamp() WHERE id=%s", + (final_code, job['attachment_id'])) + conn.execute('UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL,last_error=%s WHERE id=%s', (code, job['id'])) + event(conn, job, 'failed', final_code) + telemetry.event('job.failed', 'ERROR', job_id=str(job['id']), attempt=job['attempt'], error_code=final_code) + return 'failed' + delay = random.uniform(0.5, 1.5) * min(60, BACKOFF * 2 ** min(job['attempt'] - 1, 10)) + conn.execute('''UPDATE attachment_jobs SET attempt=attempt+1, lease_token=NULL,lease_until=NULL, + last_error=%s,next_attempt_at=clock_timestamp()+%s*interval '1 second' WHERE id=%s''', (code, delay, job['id'])) + conn.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", (job['attachment_id'],)) + conn.execute('''INSERT INTO attachment_outbox(job_id,attempt,available_at) + SELECT id,attempt,next_attempt_at FROM attachment_jobs WHERE id=%s''', (job['id'],)) + event(conn, job, 'retry', code) + context = telemetry.carrier() + if context.get('traceparent'): + conn.execute('UPDATE attachment_jobs SET traceparent=%s,tracestate=%s WHERE id=%s', + (context['traceparent'], context.get('tracestate'), job['id'])) + telemetry.event('job.retry', 'ERROR', job_id=str(job['id']), attempt=job['attempt'], error_code=code) + return 'retry' + + +def fail(job, code, permanent=False): + with db.connection() as conn: + current = locked(conn, job['id']) + if current['status'] == 'processing' and current['lease_token'] == job['lease_token']: + return retry_or_fail(conn, current, code, permanent) + return 'obsolete' + + +def recover(): + with db.connection() as conn: + rows = conn.execute('''SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id + WHERE a.status='processing' AND j.lease_until size: + last = items[size - 1] + value = {"scope": scope, "created_at": last["created_at"], "id": last["id"]} + cursor = base64.urlsafe_b64encode(json.dumps(value).encode()).decode() + return {"items": items[:size], "next_cursor": cursor} diff --git a/src/passwords.py b/src/passwords.py new file mode 100644 index 0000000..bc98066 --- /dev/null +++ b/src/passwords.py @@ -0,0 +1,22 @@ +"""Argon2id. Одновременно считаются не более двух хешей.""" +from threading import Semaphore + +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError + + +hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +slots = Semaphore(2) + + +def hash_password(password): + with slots: + return hasher.hash(password) + + +def matches(encoded, password): + with slots: + try: + return hasher.verify(encoded, password) + except VerificationError: + return False diff --git a/src/publisher.py b/src/publisher.py new file mode 100644 index 0000000..0e155db --- /dev/null +++ b/src/publisher.py @@ -0,0 +1,46 @@ +"""Outbox dispatcher: повтор XADD при неопределённом исходе безопасен.""" +import logging +import time + +from src import broker, db, faults, jobs, telemetry + + +def publish_one(client): + with db.connection() as conn: + job = conn.execute('''SELECT j.*,o.attempt AS delivery_attempt FROM attachment_outbox o + JOIN attachment_jobs j ON j.id=o.job_id WHERE o.published_at IS NULL + AND o.available_at<=clock_timestamp() ORDER BY o.available_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED''').fetchone() + if not job: + return False + job['attempt'] = job['delivery_attempt'] + faults.hit(job['id'], 'before_publish') + broker.publish(client, job) + faults.hit(job['id'], 'after_publish') + conn.execute('UPDATE attachment_outbox SET published_at=clock_timestamp() WHERE job_id=%s AND attempt=%s', + (job['id'], job['attempt'])) + return True + + +if __name__ == '__main__': + telemetry.setup('messenger-publisher', db.pool, queue=True) + db.pool.open(wait=True, timeout=30) + client = broker.connect() + last_observation = 0 + try: + while True: + try: + if time.monotonic() - last_observation >= 5: + with db.connection() as conn: + telemetry.observe_queue(conn) + last_observation = time.monotonic() + jobs.recover() + broker.bootstrap(client) + if not publish_one(client): + time.sleep(0.25) + except Exception as error: + logging.error('Outbox postponed: %s', type(error).__name__) + time.sleep(1) + finally: + client.close() + db.pool.close() + telemetry.shutdown() diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 0000000..4bf40f7 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,156 @@ +"""Параметризованный SQL. Одна функция записи — одна транзакция.""" +from uuid import UUID, uuid4 + +from psycopg.errors import ForeignKeyViolation, UniqueViolation + +from src import db, pagination +from src.errors import ApiError +from src.config import IDEMPOTENCY_TTL + + +def utc(value): + return value.isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def public_user(row): + return {"id": str(row["id"]), "username": row["username"], + "display_name": row["display_name"], "created_at": utc(row["created_at"])} + + +def public_chat(row): + return {"id": str(row["id"]), "title": row["title"], + "member_ids": [str(value) for value in row["member_ids"]], "created_at": utc(row["created_at"])} + + +def public_message(row): + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), + "text": row["text"], "created_at": utc(row["created_at"]), + "attachment_ids": [str(value) for value in row.get("attachment_ids", [])]} + + +MESSAGE_ATTACHMENTS = """ARRAY(SELECT attachment_id FROM message_attachments + WHERE message_id=m.id ORDER BY position) AS attachment_ids""" + + +def create_user(username, display_name, password_hash): + try: + with db.connection() as conn: + row = conn.execute("""INSERT INTO users(id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) RETURNING *""", (uuid4(), username, display_name, password_hash)).fetchone() + result = public_user(row) + return result + except UniqueViolation: + raise ApiError(409, "conflict", "Имя пользователя уже занято") from None + + +def find_login(username): + with db.connection() as conn: + return conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone() + + +def find_user(user_id): + with db.connection() as conn: + row = conn.execute("SELECT * FROM users WHERE id=%s", (UUID(user_id),)).fetchone() + return public_user(row) if row else None + + +def create_chat(user_id, title, member_ids): + if user_id not in member_ids: + raise ApiError(400, "invalid_request", "Создатель должен входить в member_ids") + try: + with db.connection() as conn: + row = conn.execute("INSERT INTO chats(id, title) VALUES (%s, %s) RETURNING *", (uuid4(), title)).fetchone() + with conn.cursor() as cur: + cur.executemany("INSERT INTO chat_members(chat_id, user_id) VALUES (%s, %s)", + [(row["id"], UUID(member)) for member in member_ids]) + row["member_ids"] = member_ids + result = public_chat(row) + return result + except ForeignKeyViolation: + raise ApiError(400, "invalid_request", "Все участники должны существовать") from None + + +def require_member(conn, user_id, chat_id): + row = conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (UUID(chat_id), UUID(user_id))).fetchone() + if row is None: + raise ApiError(404, "not_found", "Чат не найден") + + +def get_chat(user_id, chat_id): + with db.connection() as conn: + require_member(conn, user_id, chat_id) + row = conn.execute("""SELECT c.*, ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id) AS member_ids + FROM chats c WHERE c.id=%s""", (UUID(chat_id),)).fetchone() + return public_chat(row) + + +def list_chats(user_id, limit, cursor): + scope = "chats:" + user_id + size, anchor = pagination.parameters(limit, cursor, scope) + with db.connection() as conn: + if anchor: + found = conn.execute("""SELECT 1 FROM chats c JOIN chat_members m ON m.chat_id=c.id + WHERE m.user_id=%s AND (c.created_at, c.id)=(%s, %s)""", (UUID(user_id), *anchor)).fetchone() + if not found: + raise ApiError(400, "invalid_request", "Неверный cursor") + condition = "AND (c.created_at, c.id) > (%s, %s)" if anchor else "" + params = (UUID(user_id), *(anchor or ()), size + 1) + rows = conn.execute("""WITH selected AS ( + SELECT c.* FROM chats c JOIN chat_members own ON own.chat_id=c.id + WHERE own.user_id=%s """ + condition + """ ORDER BY c.created_at, c.id LIMIT %s + ) SELECT c.*, ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id) AS member_ids + FROM selected c ORDER BY c.created_at, c.id""", params).fetchall() + return pagination.page([public_chat(row) for row in rows], size, scope) + + +def create_message(user_id, chat_id, text, key, attachment_ids=()): + attachment_ids = [UUID(value) for value in attachment_ids] + with db.connection() as conn: + require_member(conn, user_id, chat_id) + # Одна блокировка на область ключа во всех API-процессах. + # Побочный эффект и результат повтора фиксируются одной транзакцией. + scope = user_id + ":POST:/api/v1/chats/" + chat_id + "/messages:" + key + conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (scope,)) + previous = conn.execute("SELECT m.*, i.request_text, i.request_attachment_ids, " + MESSAGE_ATTACHMENTS + """ FROM message_idempotency i + JOIN messages m ON m.id=i.message_id + WHERE i.user_id=%s AND i.chat_id=%s AND i.key=%s AND i.expires_at>clock_timestamp()""", + (UUID(user_id), UUID(chat_id), key)).fetchone() + if previous: + if previous["request_text"] != text or previous["request_attachment_ids"] != attachment_ids: + raise ApiError(409, "conflict", "Этот ключ уже использован с другим сообщением") + return public_message(previous) + if attachment_ids: + available = conn.execute("SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s)", + (UUID(chat_id), attachment_ids)).fetchall() + if len(available) != len(attachment_ids): + raise ApiError(404, "not_found", "Вложение не найдено в этом чате") + if any(item["status"] == "failed" for item in available): + raise ApiError(409, "conflict", "Вложение не обработано") + conn.execute("DELETE FROM message_idempotency WHERE user_id=%s AND chat_id=%s AND key=%s", + (UUID(user_id), UUID(chat_id), key)) + row = conn.execute("""INSERT INTO messages(id, chat_id, sender_id, text) VALUES (%s, %s, %s, %s) + RETURNING *""", (uuid4(), UUID(chat_id), UUID(user_id), text)).fetchone() + for position, attachment_id in enumerate(attachment_ids): + conn.execute("INSERT INTO message_attachments(message_id, attachment_id, position) VALUES (%s, %s, %s)", + (row["id"], attachment_id, position)) + conn.execute("""INSERT INTO message_idempotency(user_id, chat_id, key, request_text, request_attachment_ids, message_id, expires_at) + VALUES (%s, %s, %s, %s, %s, %s, clock_timestamp() + %s * interval '1 second')""", + (UUID(user_id), UUID(chat_id), key, text, attachment_ids, row["id"], IDEMPOTENCY_TTL)) + row["attachment_ids"] = attachment_ids + result = public_message(row) + return result + + +def list_messages(user_id, chat_id, limit, cursor): + scope = "messages:" + chat_id + size, anchor = pagination.parameters(limit, cursor, scope) + with db.connection() as conn: + require_member(conn, user_id, chat_id) + if anchor: + found = conn.execute("SELECT 1 FROM messages WHERE chat_id=%s AND (created_at, id)=(%s, %s)", (UUID(chat_id), *anchor)).fetchone() + if not found: + raise ApiError(400, "invalid_request", "Неверный cursor") + condition = "AND (m.created_at, m.id) > (%s, %s)" if anchor else "" + rows = conn.execute("SELECT m.*, " + MESSAGE_ATTACHMENTS + " FROM messages m WHERE m.chat_id=%s " + condition + + " ORDER BY m.created_at, m.id LIMIT %s", (UUID(chat_id), *(anchor or ()), size + 1)).fetchall() + return pagination.page([public_message(row) for row in rows], size, scope) diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..af767c9 --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,148 @@ +"""Explicit, bounded OTel SDK instrumentation. Never exports bodies, SQL or credentials.""" +from contextlib import contextmanager +from datetime import datetime, timezone +import json +import logging +import os +import socket +import sys +from time import perf_counter + +from opentelemetry import metrics, trace +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from opentelemetry.trace import SpanKind, Status, StatusCode + +propagator = TraceContextTextMapPropagator() +tracer = trace.get_tracer('messenger', '6.0.0') +meter = metrics.get_meter('messenger', '6.0.0') +http_requests = meter.create_counter('messenger.http.requests') +http_duration = meter.create_histogram('messenger.http.duration', unit='s') +job_attempts = meter.create_counter('messenger.job.attempts') +job_duration = meter.create_histogram('messenger.job.duration', unit='s') +ready_duration = meter.create_histogram('messenger.attachment.ready.duration', unit='s') +busy = meter.create_up_down_counter('messenger.worker.busy') +state = {'depth': 0, 'oldest': 0, 'good': 0, 'bad': 0} +providers = [] +service_name = 'messenger' +otel_logger = None + + +def setup(name, pool=None, worker=False, queue=False): + global service_name, otel_logger + service_name = name + if os.environ.get('OTEL_SDK_DISABLED', 'false').lower() == 'true': + return + import psutil + from opentelemetry._logs import set_logger_provider + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + resource = Resource.create({ + 'service.name': name, 'service.version': '6.0.0', + 'service.instance.id': socket.gethostname() + ':' + str(os.getpid()), + 'deployment.environment.name': 'lab', + }) + endpoint = os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/') + reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint + '/v1/metrics', timeout=2), + export_interval_millis=5000, export_timeout_millis=2500) + bounds = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30, 60, 120, 300, 600] + mp = MeterProvider(resource=resource, metric_readers=[reader], views=[ + View(instrument_type=metrics.Histogram, aggregation=ExplicitBucketHistogramAggregation(bounds))]) + metrics.set_meter_provider(mp) + tp = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(1.0))) + tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint + '/v1/traces', timeout=2), + max_queue_size=256, max_export_batch_size=64, schedule_delay_millis=1000, meter_provider=mp)) + trace.set_tracer_provider(tp) + lp = LoggerProvider(resource=resource) + lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint + '/v1/logs', timeout=2), + max_queue_size=256, max_export_batch_size=64, schedule_delay_millis=1000, meter_provider=mp)) + set_logger_provider(lp) + otel_logger = lp.get_logger('messenger', '6.0.0') + providers.extend([tp, mp, lp]) + process = psutil.Process() + meter.create_observable_gauge('messenger.process.memory', unit='By', callbacks=[ + lambda _: [metrics.Observation(process.memory_info().rss)]]) + meter.create_observable_counter('messenger.process.cpu.time', unit='s', callbacks=[ + lambda _: [metrics.Observation(sum(process.cpu_times()[:2]))]]) + if pool is not None: + def pool_stats(_): + s = pool.get_stats() + return [metrics.Observation(s.get('pool_size', 0) - s.get('pool_available', 0), {'state': 'used'}), + metrics.Observation(s.get('requests_waiting', 0), {'state': 'waiting'})] + meter.create_observable_gauge('messenger.db.pool', callbacks=[pool_stats]) + if worker: + meter.create_observable_gauge('messenger.worker.capacity', callbacks=[ + lambda _: [metrics.Observation(1)]]) + if queue: + attrs = {'queue': 'attachments'} + meter.create_observable_gauge('messenger.queue.depth', callbacks=[ + lambda _: [metrics.Observation(state['depth'], attrs)]]) + meter.create_observable_gauge('messenger.queue.oldest.age', unit='s', callbacks=[ + lambda _: [metrics.Observation(state['oldest'], attrs)]]) + meter.create_observable_counter('messenger.ready.deadlines', callbacks=[ + lambda _: [metrics.Observation(state[outcome], {'outcome': outcome}) for outcome in ('good', 'bad')]]) + + +def shutdown(): + for provider in providers: + provider.shutdown() + + +def carrier(): + value = {} + propagator.inject(value) + return value + + +@contextmanager +def span(name, *, kind=SpanKind.INTERNAL, parent=None, attributes=None, require_parent=False): + if require_parent and not trace.get_current_span().get_span_context().is_valid: + yield trace.get_current_span() + return + context = propagator.extract(parent) if parent is not None else None + # Automatic exception events can contain SQL values, tokens or object keys. + with tracer.start_as_current_span(name, context=context, kind=kind, attributes=attributes, + record_exception=False, set_status_on_exception=False) as current: + try: + yield current + except Exception as error: + current.set_attribute('error.type', type(error).__name__) + current.set_status(Status(StatusCode.ERROR)) + raise + + +def event(name, severity='INFO', **fields): + current = trace.get_current_span().get_span_context() + record = {'timestamp': datetime.now(timezone.utc).isoformat(), 'severity': severity, + 'service.name': service_name, 'event': name, + 'trace_id': format(current.trace_id, '032x') if current.is_valid else '', + 'span_id': format(current.span_id, '016x') if current.is_valid else '', **fields} + # Only explicit call sites supply fields. No exception text / headers / bodies. + encoded = json.dumps(record, ensure_ascii=False) + print(encoded, file=sys.stdout, flush=True) + if otel_logger is not None: + from opentelemetry._logs import SeverityNumber + otel_logger.emit(body=encoded, severity_text=severity, + severity_number=SeverityNumber.ERROR if severity == 'ERROR' else SeverityNumber.INFO, + attributes={'event': name, **fields}) + + +def observe_queue(conn): + row = conn.execute("""SELECT count(*) FILTER (WHERE a.status IN ('queued','processing')) AS depth, + COALESCE(max(EXTRACT(epoch FROM clock_timestamp()-j.created_at)) + FILTER (WHERE a.status IN ('queued','processing')),0) AS oldest, + count(*) FILTER (WHERE j.created_at<=clock_timestamp()-interval '30 seconds' + AND j.ready_at<=j.created_at+interval '30 seconds') AS good, + count(*) FILTER (WHERE j.created_at<=clock_timestamp()-interval '30 seconds' + AND (j.ready_at IS NULL OR j.ready_at>j.created_at+interval '30 seconds')) AS bad + FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id""").fetchone() + state.update({key: float(row[key]) for key in state}) diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..c9b9c67 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,227 @@ +"""Worker supervisors: concurrency процессов, prefetch=1, дочерний resize с timeout.""" +from datetime import datetime, timezone +import hashlib +import json +import logging +import multiprocessing +import os +from pathlib import Path +import re +import signal +import subprocess +import sys +from tempfile import TemporaryDirectory +import time +from uuid import UUID, uuid4 + +from psycopg.types.json import Jsonb + +from src import broker, db, faults, jobs, object_store, telemetry +from src.errors import ApiError + + +class ProcessingError(Exception): + def __init__(self, code, permanent=False): + self.code, self.permanent = code, permanent + + +def verify_ready(job): + # Старые ready из lab4 не имеют job; у новой job всегда все три варианта. + if {v['name'] for v in job['variants']} != {'thumbnail', 'crop', 'optimized'}: + raise ProcessingError('incomplete_result') + for key, size in [(job['object_key'], job['size_bytes'])] + [ + (job['object_key'].rsplit('/', 1)[0] + '/' + v['name'], v['size_bytes']) for v in job['variants'] + ]: + with object_store.available(): + if object_store.client.stat_object(object_store.BUCKET, key).size != size: + raise ProcessingError('incomplete_result') + + +def process(job): + with TemporaryDirectory(prefix='image-job-') as directory: + output = Path(directory) + source = output / 'original' + response = object_store.get(job['staging_key']) + size, digest = 0, hashlib.sha256() + try: + with source.open('wb') as file: + while chunk := response.read(65536): + size += len(chunk) + if size > job['size_bytes'] or size > 10485760: + raise ProcessingError('invalid_image', True) + digest.update(chunk) + file.write(chunk) + finally: + response.close() + response.release_conn() + if size != job['size_bytes'] or digest.hexdigest() != job['sha256']: + raise ProcessingError('invalid_image', True) + try: + result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', str(source), str(output), str(job['id'])], + timeout=jobs.TIMEOUT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except subprocess.TimeoutExpired: + raise ProcessingError('processing_timeout') from None + if result.returncode: + raise ProcessingError('invalid_image' if result.returncode == 65 else 'resource_limit', result.returncode == 65) + variants = json.loads((output / 'variants.json').read_text()) + # Короткая транзакция fence перед side effects и публикацией результата. + with db.connection() as conn: + conn.execute("SET LOCAL idle_in_transaction_session_timeout='30s'") + current = jobs.locked(conn, job['id']) + active = conn.execute('SELECT lease_until>clock_timestamp() AS active FROM attachment_jobs WHERE id=%s', (job['id'],)).fetchone()['active'] + if current['status'] != 'processing' or current['lease_token'] != job['lease_token'] or not active: + return False + with source.open('rb') as file: + object_store.put(job['object_key'], file, job['size_bytes'], job['content_type']) + prefix = job['object_key'].rsplit('/', 1)[0] + '/' + for variant in variants: + with (output / variant['name']).open('rb') as file: + object_store.put(prefix + variant['name'], file, variant['size_bytes'], variant['content_type']) + faults.hit(job['id'], 'after_objects') + conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", (Jsonb(variants), job['attachment_id'])) + conn.execute('UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL,last_error=NULL,ready_at=clock_timestamp() WHERE id=%s', (job['id'],)) + jobs.event(conn, job, 'ready') + faults.hit(job['id'], 'before_ack') + telemetry.ready_duration.record((datetime.now(timezone.utc) - job['created_at']).total_seconds(), {'job_type': 'attachment.process.v1'}) + return True + + +def _handle(client, message_id, fields): + try: + payload = json.loads(fields['job']) + required = {'schema_version', 'type', 'job_id', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'} + if not isinstance(payload, dict) or not required <= payload.keys() or payload.keys() - required - {'traceparent', 'tracestate'}: + raise ValueError('invalid job') + if type(payload['schema_version']) is not int or payload['schema_version'] != 1 or payload['type'] != 'attachment.process.v1': + raise ValueError('invalid job') + payload['job_id'] = str(UUID(payload['job_id'])) + payload['attachment_id'] = str(UUID(payload['attachment_id'])) + if type(payload['attempt']) is not int or payload['attempt'] < 1 or type(payload['pipeline_version']) is not int or payload['pipeline_version'] != 1: + raise ValueError('invalid job') + if not isinstance(payload['created_at'], str) or datetime.fromisoformat(payload['created_at'].replace('Z', '+00:00')).tzinfo is None: + raise ValueError('invalid job') + if 'traceparent' in payload and (not isinstance(payload['traceparent'], str) or re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', payload['traceparent']) is None): + raise ValueError('invalid trace context') + if 'tracestate' in payload and (not isinstance(payload['tracestate'], str) or len(payload['tracestate']) > 512): + raise ValueError('invalid trace context') + except (ValueError, TypeError, KeyError): + logging.error('Invalid broker message rejected') + broker.acknowledge(client, message_id) + return 'invalid' + state, job = jobs.claim(payload) + if state == 'busy': + return 'busy' + if state == 'ready': + verify_ready(job) + outcome = 'duplicate' if state == 'ready' else state + if state == 'claimed': + try: + faults.hit(job['id'], 'before_process') + if not process(job): + return 'obsolete' + outcome = 'ready' + except ProcessingError as error: + outcome = jobs.fail(job, error.code, error.permanent) + except Exception: + outcome = jobs.fail(job, 'storage_unavailable') + # ready/failed/obsolete или записанный retry; при потере COMMIT исключение не даёт ACK. + broker.acknowledge(client, message_id) + return outcome + + +def handle(client, message_id, fields): + try: + payload = json.loads(fields.get('job', '{}')) + parent = {key: payload[key] for key in ('traceparent', 'tracestate') if isinstance(payload.get(key), str)} + job_id = str(UUID(payload['job_id'])) + attempt = payload.get('attempt', 1) + if type(attempt) is not int or not 1 <= attempt <= 1000000: + attempt = 1 + except (ValueError, TypeError, KeyError, AttributeError): + parent, job_id, attempt = {}, '', 1 + started = time.perf_counter() + with telemetry.span('attachments process', kind=telemetry.SpanKind.CONSUMER, parent=parent, + attributes={'messaging.system': 'redis', 'messaging.destination.name': broker.STREAM, + 'messaging.operation.type': 'process', 'job.attempt': attempt}) as current: + telemetry.busy.add(1) + outcome = 'error' + try: + outcome = _handle(client, message_id, fields) + finally: + telemetry.busy.add(-1) + attrs = {'job_type': 'attachment.process.v1', 'outcome': outcome} + telemetry.job_attempts.add(1, attrs) + telemetry.job_duration.record(time.perf_counter() - started, attrs) + current.set_attribute('job.outcome', outcome) + if outcome in {'failed', 'retry', 'error', 'invalid'}: + current.set_status(telemetry.Status(telemetry.StatusCode.ERROR)) + telemetry.event('job.finished', 'ERROR' if outcome in {'failed', 'retry', 'error'} else 'INFO', + job_id=job_id, attempt=attempt, outcome=outcome) + + +def consume(): + telemetry.setup('messenger-worker', db.pool, worker=True) + db.pool.open(wait=True, timeout=30) + client = broker.connect() + consumer = 'worker-' + uuid4().hex + cursor = '0-0' + try: + while True: + try: + broker.bootstrap(client) + claimed = client.xautoclaim(broker.STREAM, broker.GROUP, consumer, + min_idle_time=jobs.LEASE * 1000, start_id=cursor, count=1) + cursor = claimed[0] + entries = claimed[1] + if not entries: + batches = client.xreadgroup(broker.GROUP, consumer, {broker.STREAM: '>'}, count=1, block=1000) + entries = batches[0][1] if batches else [] + for message_id, fields in entries: + handle(client, message_id, fields) + except Exception as error: + logging.error('Worker postponed: %s', type(error).__name__) + time.sleep(1) + finally: + client.close() + db.pool.close() + telemetry.shutdown() + + +def main(): + concurrency = int(os.environ.get('WORKER_CONCURRENCY', '2')) + if not 1 <= concurrency <= 2: + raise ValueError('WORKER_CONCURRENCY: 1 или 2 для заданного memory limit') + context = multiprocessing.get_context('spawn') + processes = [] + stopping = False + def stop(signum, frame): + nonlocal stopping + stopping = True + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + try: + while not stopping: + alive = [] + for child in processes: + if child.is_alive(): + alive.append(child) + else: + child.join() + processes = alive + while len(processes) < concurrency: + child = context.Process(target=consume) + child.start() + processes.append(child) + time.sleep(0.25) + finally: + for child in processes: + child.terminate() + for child in processes: + child.join(timeout=5) + if child.is_alive(): + child.kill() + child.join() + + +if __name__ == '__main__': + main() diff --git a/student-tests/attachment_checks.py b/student-tests/attachment_checks.py new file mode 100644 index 0000000..63562be --- /dev/null +++ b/student-tests/attachment_checks.py @@ -0,0 +1,275 @@ +"""Реальные отказы/очистка/память уровня 4; только временный тестовый стенд.""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import io +import json +import os +import socket +import subprocess +import threading +import time +from urllib.parse import urlsplit +from uuid import uuid4 + +from PIL import Image + +from helpers import Client, ROOT +from client import Failure + + +def run_checks(base, env, compose, apis, script, sql): + result = {} + client = Client(base, timeout=30) + alice, bob, _, chat = client.bootstrap() + png = (ROOT / "tests/fixtures/sample.png").read_bytes() + upload_path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png" + + def upload(): + return client.call("POST", upload_path, 201, actor=alice, raw=png, + headers={"Content-Type": "image/png"}, schema="Attachment") + + def inside(code, data=None): + completed = subprocess.run(["docker", "exec", "-i", apis()[0]["Id"], "python", "-c", code], + input=json.dumps(data), capture_output=True, text=True, timeout=45, check=True) + return json.loads(completed.stdout) + + def cleanup(): + return inside("from src import db; from src.cleanup import cleanup_once; import json; " + "db.pool.open(wait=True); print(json.dumps(cleanup_once())); db.pool.close()") + + def stat(key): + return inside("import json,sys; from src import object_store; from minio.error import S3Error\n" + "key=json.load(sys.stdin)\n" + "try:\n r=object_store.client.stat_object(object_store.BUCKET,key); print(json.dumps({'size':r.size}))\n" + "except S3Error as e:\n print(json.dumps({'error':e.code}))", key) + + def age(key): + sql("UPDATE attachment_uploads SET created_at=clock_timestamp()-interval '2 hours' WHERE object_key='" + key + "'") + + compose("stop", "cleanup") + try: + for _ in range(2): + compose("run", "--rm", "s3-bootstrap") + ready = upload()["json"] + key = sql("SELECT object_key FROM attachments WHERE id='" + ready["id"] + "'") + backend_id = compose("ps", "-q", "s3-storage") + backend = json.loads(subprocess.check_output(["docker", "inspect", backend_id], text=True))[0] + assert not set(backend["NetworkSettings"]["Networks"]) & set(apis()[0]["NetworkSettings"]["Networks"]) + backend_ip = next(iter(backend["NetworkSettings"]["Networks"].values()))["IPAddress"] + # Вызов S3 напрямую из контейнера клиента, минуя HTTP API приложения. + result["private_bucket"] = inside(""" +import io,json,socket,sys +from urllib.error import HTTPError +from urllib.request import urlopen +from minio.error import S3Error +from src import object_store +data=json.load(sys.stdin); key=data['key'] +try: + urlopen('http://s3:9000/'+object_store.BUCKET+'/'+key,timeout=3) + raise AssertionError('public original') +except HTTPError as e: + assert e.code == 403 +for operation in (lambda: object_store.client.make_bucket('forbidden-bucket'), + lambda: object_store.client.put_object(object_store.BUCKET,'outside-prefix',io.BytesIO(b'x'),1)): + try: + operation() + raise AssertionError('excess application permissions') + except S3Error as e: + assert e.code == 'AccessDenied', e.code +for port in (9333,8888,8080): + with socket.socket() as connection: + connection.settimeout(1) + assert connection.connect_ex(('s3',port)) != 0, 'admin/backend port open' + with socket.socket() as connection: + connection.settimeout(.5) + assert connection.connect_ex((data['backend_ip'],port)) != 0, 'storage network reachable from API' +print(json.dumps({'anonymous_get':403,'create_bucket':'AccessDenied','write_outside_prefix':'AccessDenied', + 'admin_ports_unreachable':[9333,8888,8080],'storage_network_isolated':True,'bootstrap_runs':2})) +""", {"key":key, "backend_ip":backend_ip}) + print("PASS: private S3, limited credentials, isolated administration, repeated bootstrap", flush=True) + + # Останавливаем именно реплику, которая ответила на upload, а затем читаем другой. + uploaded = upload() + target = uploaded["headers"]["X-Instance-Id"] + stopped_at = time.monotonic() + subprocess.run([str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, + env=dict(env, TARGET_INSTANCE_ID=target), check=True, timeout=15) + script("wait-ready.sh") + while True: + try: + response = client.call("GET", "/api/v1/attachments/" + uploaded["json"]["id"] + "/content", actor=bob) + break + except Failure: + assert time.monotonic() - stopped_at < 10, "download failover exceeded 10 seconds" + time.sleep(.1) + assert response["bytes"] == png and response["headers"]["X-Instance-Id"] != target + stopped = next(info for info in apis() if info["Config"]["Hostname"] == target) + assert not stopped["State"]["Running"] + result["upload_replica_failover"] = {"uploaded_by": target, "stopped_container": stopped["Id"], + "downloaded_by": response["headers"]["X-Instance-Id"], "sha256": hashlib.sha256(png).hexdigest(), + "recovery_seconds":round(time.monotonic()-stopped_at,3)} + compose("start", "api") + script("wait-ready.sh") + print("PASS: original downloaded through another replica after stopping uploader", flush=True) + + # Незавершённое HTTP-тело не достигает S3 и не создаёт метаданных/журнала. + before = sql("SELECT (SELECT count(*) FROM attachments)||':'||(SELECT count(*) FROM attachment_uploads)") + address = urlsplit(base) + with socket.create_connection((address.hostname, address.port), timeout=3) as connection: + headers = ("POST " + upload_path + " HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer " + alice["token"] + + "\r\nContent-Type: image/png\r\nContent-Length: 1000000\r\n\r\n").encode() + connection.sendall(headers + png[:30]) + time.sleep(0.5) + assert sql("SELECT (SELECT count(*) FROM attachments)||':'||(SELECT count(*) FROM attachment_uploads)") == before + result["interrupted_body"] = {"metadata_or_journal_created": False} + + # Детерминированный реальный отказ PostgreSQL ПОСЛЕ успешного S3 PUT. + sql("""CREATE FUNCTION lab4_reject_attachment() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'injected storage failure' USING ERRCODE='08006'; END $$; + CREATE TRIGGER lab4_reject BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION lab4_reject_attachment();""") + try: + failure = client.call("POST", upload_path, 503, actor=alice, raw=png, headers={"Content-Type": "image/png"}) + assert failure["json"]["error"]["code"] == "unavailable" + finally: + sql("DROP TRIGGER lab4_reject ON attachments; DROP FUNCTION lab4_reject_attachment()") + pending_key = sql("SELECT object_key FROM attachment_uploads ORDER BY created_at DESC LIMIT 1") + assert stat(pending_key)["size"] == len(png) + assert sql("SELECT count(*) FROM attachments WHERE object_key='" + pending_key + "'") == "0" + assert cleanup()["removed"] == 0 + age(pending_key) + lock_code = ("from src import db; from src.attachments import lock_upload; import sys\n" + "db.pool.open(wait=True)\nwith db.connection() as c:\n" + " lock_upload(c,'" + pending_key + "'); print('locked',flush=True); sys.stdin.readline()\n" + "db.pool.close()") + holder = subprocess.Popen(["docker", "exec", "-i", apis()[0]["Id"], "python", "-u", "-c", lock_code], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + assert holder.stdout.readline().strip() == "locked" + assert cleanup()["skipped_active"] == 1 + assert stat(pending_key)["size"] == len(png) + finally: + holder.communicate("\n", timeout=10) + assert cleanup()["removed"] == 1 + assert stat(pending_key).get("error") == "NoSuchKey" + assert cleanup()["removed"] == 0 + # Даже устаревшая ошибочная запись журнала не позволяет удалить ready-файл. + sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + str(uuid4()) + "','" + key + + "',clock_timestamp()-interval '2 hours')") + assert cleanup()["kept_ready"] == 1 + assert client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob)["bytes"] == png + result["compensation"] = {"failure_after_put":503, "orphan_recorded":True, "fresh_kept":True, + "active_locked_kept":True, "expired_removed":True, "ready_kept":True, "repeat_safe":True} + print("PASS: post-PUT DB failure recorded; cleanup protects fresh, active and ready uploads", flush=True) + + try: + compose("stop", "--timeout", "8", "s3") + result["s3_outage"] = inside(""" +import json,sys +from urllib.request import Request,urlopen +from urllib.error import HTTPError +data=json.load(sys.stdin) +statuses={} +for name,method,path,body in [('live','GET','/health/live',None),('ready','GET','/health/ready',None), + ('download','GET','/api/v1/attachments/'+data['id']+'/content',None), + ('upload','POST',data['upload'],bytes.fromhex(data['png']))]: + req=Request('http://127.0.0.1:8080'+path,data=body,method=method, + headers={'Authorization':'Bearer '+data['token'],'Content-Type':'image/png'}) + try: response=urlopen(req,timeout=8) + except HTTPError as error: response=error + with response: + statuses[name]=response.status + payload=json.load(response) + if name!='live': + assert response.status==503 and payload['error']['code']=='unavailable' + assert payload['error']['request_id']==response.headers['X-Request-Id'] +assert statuses['live']==200 +print(json.dumps(statuses)) +""", {"id":ready["id"], "token":alice["token"], "upload":upload_path, "png":png.hex()}) + finally: + compose("start", "s3") + script("wait-ready.sh") + assert client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob)["bytes"] == png + sql("UPDATE attachment_uploads SET created_at=clock_timestamp()-interval '2 hours'") + assert cleanup()["removed"] >= 1 + result["s3_outage"]["original_preserved_after_recovery"] = True + print("PASS: S3 outage gives API 503; recovery preserves original and session", flush=True) + + # Доказываем работу постоянного worker, а не только ручного вызова cleanup_once. + inside("import io,json; from src import object_store; " + "object_store.put('attachments/cleanup-worker-test',io.BytesIO(b'x'),1,'application/octet-stream'); print('{}')") + sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + str(uuid4()) + + "','attachments/cleanup-worker-test',clock_timestamp()-interval '2 hours')") + compose("start", "cleanup") + deadline = time.monotonic() + 40 + while sql("SELECT count(*) FROM attachment_uploads WHERE object_key='attachments/cleanup-worker-test'") != "0": + assert time.monotonic() < deadline, "automatic cleanup not running" + time.sleep(0.2) + assert stat('attachments/cleanup-worker-test').get("error") == "NoSuchKey" + result["compensation"]["automatic_worker"] = True + + # 10 одновременно отправляемых реальных PNG ~5.5 MiB; монитор cgroup каждые 20 ms. + image = Image.frombytes("RGB", (1600, 1200), os.urandom(1600 * 1200 * 3)) + buffer = io.BytesIO() + image.save(buffer, format="PNG") + image.close() + data = buffer.getvalue() + expected_sha = hashlib.sha256(data).hexdigest() + monitor_code = """ +from pathlib import Path +import json,sys,threading,time +root=Path('/sys/fs/cgroup'); stop=threading.Event(); samples=[] +threading.Thread(target=lambda:(sys.stdin.readline(),stop.set()),daemon=True).start() +start=time.monotonic(); before=int((root/'memory.current').read_text()) +print('ready',flush=True) +while not stop.is_set(): + samples.append([round(time.monotonic()-start,3),int((root/'memory.current').read_text())]) + stop.wait(.02) +print(json.dumps({'baseline_bytes':before,'sample_peak_bytes':max(s[1] for s in samples), + 'lifetime_peak_bytes':int((root/'memory.peak').read_text()),'samples':samples}),flush=True) +""" + monitors = [] + responses = [] + try: + for info in apis(): + process = subprocess.Popen(["docker", "exec", "-i", info["Id"], "python", "-u", "-c", monitor_code], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + assert process.stdout.readline().strip() == "ready" + monitors.append((info, process)) + barrier = threading.Barrier(10) + def send(_): + concurrent_client = Client(base, timeout=60) + barrier.wait(timeout=15) + return concurrent_client.call("POST", upload_path, 201, actor=alice, raw=data, + headers={"Content-Type":"image/png"}, schema="Attachment") + started = time.monotonic() + with ThreadPoolExecutor(max_workers=10) as pool: + responses = list(pool.map(send, range(10))) + elapsed = time.monotonic() - started + finally: + memory = [] + for info, process in monitors: + output, error = process.communicate("\n", timeout=10) + assert process.returncode == 0, error + measurement = json.loads(output) + measurement.update({"instance": info["Config"]["Hostname"], "limit_bytes": info["HostConfig"]["Memory"]}) + assert measurement["sample_peak_bytes"] < measurement["limit_bytes"] + memory.append(measurement) + assert len(responses) == 10 + assert len({r["headers"]["X-Instance-Id"] for r in responses}) >= 2 + for response in responses: + assert response["json"]["sha256"] == expected_sha and response["json"]["size_bytes"] == len(data) + original = client.call("GET", "/api/v1/attachments/" + response["json"]["id"] + "/content", actor=bob) + assert hashlib.sha256(original["bytes"]).hexdigest() == expected_sha + assert all(not i["State"]["OOMKilled"] for i in apis()) + timings = sorted(r["elapsed"] * 1000 for r in responses) + result["parallel_uploads"] = {"concurrency":10, "successful":10, "errors":0, + "dimensions":[1600,1200], "size_bytes":len(data), "sha256":expected_sha, + "duration_seconds":round(elapsed,3), "successful_rps":round(10/elapsed,3), + "p50_ms":round(timings[4],3), "p95_ms":round(timings[-1],3), + "memory_sample_interval_seconds":0.02, "memory_includes_monitor_process":True, + "memory":memory, "all_original_checksums_verified":True, "oom_killed":False} + print("PASS: 10 concurrent real PNG uploads; all checksums match; memory peaks:", + [m["sample_peak_bytes"] for m in memory], flush=True) + return result + finally: + compose("start", "s3", "api", "cleanup") diff --git a/student-tests/docker_lifecycle.py b/student-tests/docker_lifecycle.py new file mode 100644 index 0000000..4800ec6 --- /dev/null +++ b/student-tests/docker_lifecycle.py @@ -0,0 +1,319 @@ +"""Проверки уровня 4 в отдельном Compose project с синтетическими данными.""" +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import json +import os +import platform +import socket +import subprocess +import sys +import threading +import time +from urllib.error import HTTPError +from urllib.request import urlopen +from uuid import uuid4 + +from helpers import Client, ROOT +from client import Failure +from resilience import history +from attachment_checks import run_checks +from postgres_pause import check_postgres_pause + + +def docker(*args): + return subprocess.check_output(["docker", *args], text=True, timeout=180).strip() + + +def main(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + project = "lab4-check-" + uuid4().hex[:10] + base = f"http://127.0.0.1:{port}" + env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), + API_REPLICAS="2", SESSION_TTL_SECONDS="3600", BASE_URL=base) + result = {"utc": datetime.now(timezone.utc).isoformat(), "project": project, + "base_url": base, "host": platform.system() + " " + platform.machine(), + "docker": docker("info", "--format", "{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}"), + "commit": docker("compose", "version", "--short"), + "profile": {"workers": 4, "pause_seconds": 0.05, "operation": "POST message then GET history, same-key retry after uncertain POST"}} + result["compose_version"] = result.pop("commit") + result["source_commit"] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + result["working_tree"] = True + + def compose(*args): + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, env=env, text=True, timeout=180).strip() + + def script(name): + subprocess.run([str(ROOT / "scripts" / name)], cwd=ROOT, env=env, check=True, timeout=240) + + def inspect_all(): + ids = compose("ps", "-a", "-q").split() + return json.loads(docker("inspect", *ids)) + + def apis(): + return [i for i in inspect_all() if i["Config"]["Labels"]["com.docker.compose.service"] == "api"] + + def metadata(info): + return {"id": info["Id"], "instance": info["Config"]["Hostname"], + "service": info["Config"]["Labels"]["com.docker.compose.service"], + "image": info["Image"], "started_at": info["State"]["StartedAt"], + "running": info["State"]["Running"], "exit_code": info["State"]["ExitCode"], + "memory_bytes": info["HostConfig"]["Memory"], "nano_cpus": info["HostConfig"]["NanoCpus"], + "published_ports": info["HostConfig"]["PortBindings"], + "mounts": [{"type": m["Type"], "destination": m["Destination"], "name": m.get("Name")} for m in info["Mounts"]]} + + def sql(query): + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query) + + def check_shutdown(): + info = apis()[0] + # Внутреннее подключение через docker exec не публикует порт API на хост. + pending_code = """import socket,time +s=socket.create_connection(('127.0.0.1',8080),timeout=2) +s.sendall(b'POST /api/v1/users HTTP/1.1\\r\\nHost: localhost\\r\\nContent-Type: application/json\\r\\nContent-Length: 1000\\r\\nExpect: 100-continue\\r\\n\\r\\n') +assert b'100 Continue' in s.recv(1024) +s.sendall(b'{"username":') +print('pending',flush=True) +time.sleep(30) +""" + pending = subprocess.Popen(["docker", "exec", info["Id"], "python", "-u", "-c", pending_code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + assert pending.stdout.readline().strip() == "pending" + started = time.monotonic() + docker("kill", "--signal=TERM", info["Id"]) + first_refusal = None + while True: + state = json.loads(docker("inspect", info["Id"]))[0]["State"] + if not state["Running"]: + break + assert time.monotonic() - started < 8 + probe = subprocess.run(["docker", "exec", info["Id"], "python", "-c", + "import socket; s=socket.socket(); s.settimeout(.2); raise SystemExit(s.connect_ex(('127.0.0.1',8080)))"], + capture_output=True, timeout=3) + if probe.returncode and first_refusal is None: + first_refusal = time.monotonic() - started + time.sleep(0.1) + elapsed = time.monotonic() - started + assert first_refusal is not None and first_refusal < elapsed < 8 + assert state["ExitCode"] == 0 and not state["OOMKilled"] + logs = subprocess.run(["docker", "logs", info["Id"]], capture_output=True, text=True, check=True) + assert "Application shutdown complete" in logs.stdout + logs.stderr + result["sigterm"] = {"instance": info["Config"]["Hostname"], "unfinished_request": True, + "shutdown_seconds": round(elapsed, 3), "new_connections_refused_seconds": round(first_refusal, 3), + "exit_code": state["ExitCode"], "oom_killed": state["OOMKilled"]} + print("PASS: SIGTERM with unfinished request:", json.dumps(result["sigterm"]), flush=True) + finally: + pending.communicate(timeout=10) + compose("start", "api") + script("wait-ready.sh") + + def load_action(label, action, target=None): + client = Client(base, timeout=2) + alice, bob, _, chat = client.bootstrap() + original = client.message(alice, chat, "До воздействия")["json"] + stop = threading.Event() + lock = threading.Lock() + successes, errors, confirmed = [], [], {} + started = time.monotonic() + + def worker(index): + c = Client(base, timeout=2) + key = uuid4().hex + while not stop.is_set(): + for method in ("POST", "GET"): + try: + if method == "POST": + response = c.message(alice, chat, "load-" + key, key=key) + with lock: + confirmed[response["json"]["id"]] = response["json"] + key = uuid4().hex + else: + response = c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", + actor=bob, schema="MessagePage") + with lock: + successes.append((time.monotonic() - started, method, + response["headers"]["X-Instance-Id"], response["elapsed"])) + except Failure as exc: + with lock: + errors.append((time.monotonic() - started, method, str(exc))) + stop.wait(0.05) + + with ThreadPoolExecutor(max_workers=4) as pool: + futures = [pool.submit(worker, i) for i in range(4)] + time.sleep(1) + action_start = time.monotonic() - started + try: + action_info = action() + action_end = time.monotonic() - started + # Охватывает обнаружение отказа и устойчивую работу после него. + time.sleep(max(3, 15 - (action_end - action_start))) + finally: + stop.set() + for future in futures: + future.result() + elapsed = time.monotonic() - started + items = history(client, bob, chat) + by_id = {m["id"]: m for m in items} + assert len(items) == len(by_id) == len({m["text"] for m in items}), "Повтор создал дубль" + assert by_id.get(original["id"]) == original + assert all(by_id.get(mid) == msg for mid, msg in confirmed.items()), "Потеря подтверждённого сообщения" + after = [s for s in successes if s[0] >= action_end] + assert {s[1] for s in after} == {"POST", "GET"}, "Нет чтений/записей после воздействия" + recovery = {method: round(min(s[0] for s in successes if s[0] >= action_start and s[1] == method) - action_start, 3) + for method in ("POST", "GET")} + assert max(recovery.values()) <= 10, recovery + max_gap = {} + for method in ("POST", "GET"): + moments = sorted(s[0] for s in successes if s[1] == method) + max_gap[method] = round(max(b - a for a, b in zip(moments, moments[1:])), 3) + assert max(max_gap.values()) <= 10, max_gap + if target: + assert target not in {s[2] for s in after[-20:]}, "Остановленная реплика ещё отвечает" + buckets = [] + for second in range(int(elapsed) + 1): + buckets.append({"second": second, "successes": sum(second <= s[0] < second + 1 for s in successes), + "errors": sum(second <= e[0] < second + 1 for e in errors)}) + latencies = sorted(s[3] * 1000 for s in successes) + summary = {"label": label, "duration_seconds": round(elapsed, 3), "successful_requests": len(successes), + "successful_writes": len(confirmed), "errors": len(errors), "successful_rps": round(len(successes) / elapsed, 2), + "first_success_after_action_seconds": recovery, "maximum_success_gap_seconds": max_gap, + "action_start_seconds": round(action_start, 3), "action_end_seconds": round(action_end, 3), + "p50_ms": round(latencies[len(latencies) // 2], 3), + "p95_ms": round(latencies[int(len(latencies) * .95)], 3), + "confirmed_data_preserved": True, "duplicate_messages": 0, + "instances": sorted({s[2] for s in successes}), "per_second": buckets, + "error_samples": errors[:5], "action": action_info} + print(json.dumps({k: v for k, v in summary.items() if k not in ("per_second", "error_samples", "action")}, ensure_ascii=False), flush=True) + return summary + + print("Command: ./scripts/check-docker.sh; project:", project, flush=True) + print("Docker:", result["docker"], flush=True) + try: + script("start.sh") + initial = inspect_all() + result["containers"] = [metadata(info) for info in initial] + assert len(apis()) == 2 + for info in initial: + assert info["HostConfig"]["Memory"] > 0 and info["HostConfig"]["NanoCpus"] > 0 + if info["Config"]["Labels"]["com.docker.compose.service"] != "lb": + assert not info["HostConfig"]["PortBindings"] + for info in apis(): + assert docker("exec", info["Id"], "id", "-u") == "10001" + assert all(m["Destination"].startswith("/run/secrets/") for m in info["Mounts"]) + print("PASS: 2 real API containers, CPU/RAM limits, API non-root and no public ports; only LB published", flush=True) + subprocess.run([str(ROOT / "scripts/check.sh")], cwd=ROOT, env=env, check=True, timeout=240) + result["api_tests"] = "public smoke + student unittest suite passed" + subprocess.run([str(ROOT / "scripts/demo.sh")], cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL, timeout=30) + assert sql("SELECT count(*) FROM schema_migrations") == "3" + count = sql("SELECT count(*) FROM messages") + for _ in range(2): + compose("run", "--rm", "migrate") + assert sql("SELECT count(*) FROM schema_migrations") == "3" + assert sql("SELECT count(*) FROM messages") == count + print("PASS: migrations only in one-shot service; repeated execution preserves data", flush=True) + result["attachments"] = run_checks(base, env, compose, apis, script, sql) + + replay_client = Client(base) + replay_actor, _, _, replay_chat = replay_client.bootstrap() + replay_key = uuid4().hex + saved_message = replay_client.message(replay_actor, replay_chat, "Повтор после рестарта", key=replay_key)["json"] + before = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]} + subprocess.run(["make", "test-persistence", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/restart.sh"], + cwd=ROOT, env=env, check=True, timeout=180) + after = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]} + assert before.keys() == after.keys() + assert all(before[cid]["started_at"] != after[cid]["started_at"] for cid in before) + assert replay_client.message(replay_actor, replay_chat, "Повтор после рестарта", key=replay_key)["json"] == saved_message + result["persistence"] = {"before": list(before.values()), "after": list(after.values()), "passed": True, + "idempotency_replay_after_restart": True} + print("PASS: both API, LB, PostgreSQL, Valkey, S3 and cleaner actually restarted; data and session preserved", flush=True) + + check_shutdown() + + dependency_client = Client(base) + actor = dependency_client.user("deps") + result["dependency_readiness"] = [] + for dependency in ("valkey", "postgres", "s3"): + try: + compose("stop", "--timeout", "8", dependency) + probe_id = apis()[0]["Id"] + readiness = subprocess.run(["docker", "exec", probe_id, "python", "-m", "src.healthcheck"], capture_output=True, timeout=10) + assert readiness.returncode == 1 + live = docker("exec", probe_id, "python", "-c", + "from urllib.request import urlopen; print(urlopen('http://127.0.0.1:8080/health/live').status)") + assert live == "200" + deadline = time.monotonic() + 8 + while True: + try: + urlopen(base + "/health/ready", timeout=2).close() + except HTTPError as error: + if error.code == 503: + break + raise + except OSError: + # Уже отправленный запрос может истечь до исключения backend. + pass + assert time.monotonic() < deadline + time.sleep(0.1) + result["dependency_readiness"].append({"dependency": dependency, "api_live": 200, "api_ready": 503, "lb_ready": 503}) + print("PASS:", dependency, "outage: API live=200, ready=503, LB ready=503", flush=True) + finally: + compose("start", dependency) + script("wait-ready.sh") + dependency_client.call("GET", "/api/v1/users/me", actor=actor, schema="User") + + result["postgres_pause"] = check_postgres_pause(compose, apis, base) + + subprocess.run(["make", "test-failover", "PYTHON=" + sys.executable, "ACTION_SCRIPT=scripts/stop-one.sh"], + cwd=ROOT, env=env, check=True, timeout=90) + assert sum(i["State"]["Running"] for i in apis()) == 1 + result["public_failover"] = {"passed": True, "containers_after": [metadata(i) for i in apis()]} + compose("start", "api") + script("wait-ready.sh") + + result["failovers"] = [] + for info in apis(): + target = info["Config"]["Hostname"] + + def stop_target(): + subprocess.run([str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, + env=dict(env, TARGET_INSTANCE_ID=target), check=True, timeout=10) + stopped = json.loads(docker("inspect", info["Id"]))[0] + assert not stopped["State"]["Running"] + assert not stopped["State"]["OOMKilled"] and stopped["State"]["ExitCode"] == 0 + # Uvicorn пишет в stderr; получаем оба потока без вывода запросов. + logs_result = subprocess.run(["docker", "logs", info["Id"]], capture_output=True, text=True, check=True) + assert "Application shutdown complete" in logs_result.stdout + logs_result.stderr + return metadata(stopped) + + result["failovers"].append(load_action("stop " + target, stop_target, target)) + compose("start", "api") + script("wait-ready.sh") + + before = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]} + + def replace(): + script("rolling-update.sh") + after = {i["Id"]: metadata(i) for i in inspect_all() if i["State"]["Running"]} + old_api = {i["id"] for i in before.values() if i["service"] == "api"} + new_api = {i["id"] for i in after.values() if i["service"] == "api"} + assert len(new_api) == 2 and not old_api & new_api + for cid, info in before.items(): + if info["service"] != "api": + assert after[cid]["started_at"] == info["started_at"] + return {"before": list(before.values()), "after": list(after.values())} + + result["rolling_update"] = load_action("sequential API replacement", replace) + result["passed"] = True + print("PASS: all lab4 grade-4 checks completed", flush=True) + finally: + output = ROOT / "evidence" / "lab4-checks.json" + output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + compose("down", "--volumes", "--timeout", "8") + print("Removed only temporary test containers and volumes:", project, flush=True) + + +if __name__ == "__main__": + main() diff --git a/student-tests/helpers.py b/student-tests/helpers.py new file mode 100644 index 0000000..d44e6fd --- /dev/null +++ b/student-tests/helpers.py @@ -0,0 +1,42 @@ +from pathlib import Path +import sys +import unittest +from urllib.parse import urlencode +from uuid import uuid4 + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client + + +class ApiTestCase(unittest.TestCase): + def setUp(self): + self.client = Client() + self.alice, self.bob, self.eve, self.chat = self.client.bootstrap() + self.path = "/api/v1/chats/" + self.chat["id"] + + def assert_error(self, method, path, status, code, **kwargs): + if method == "POST" and path.endswith("/messages"): + kwargs["headers"] = {"Idempotency-Key": uuid4().hex, **kwargs.get("headers", {})} + result = self.client.call(method, path, status, **kwargs) + self.assertEqual(result["json"]["error"]["code"], code) + return result["json"]["error"] + + def read_all(self, path, actor, schema, limit=3): + items = [] + cursor = None + seen_cursors = set() + while True: + query = {"limit": limit} + if cursor is not None: + query["cursor"] = cursor + page = self.client.call("GET", path + "?" + urlencode(query), actor=actor, schema=schema)["json"] + self.assertLessEqual(len(page["items"]), limit) + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return items + self.assertTrue(page["items"], "Пустая страница не должна вести к следующей") + self.assertNotIn(cursor, seen_cursors, "Зацикливание пагинации") + seen_cursors.add(cursor) diff --git a/student-tests/lab5_pipeline.py b/student-tests/lab5_pipeline.py new file mode 100644 index 0000000..8dd422d --- /dev/null +++ b/student-tests/lab5_pipeline.py @@ -0,0 +1,325 @@ +"""Реальный Compose-стенд: обязательный минимум и все критерии lab5 на 4.""" +from datetime import datetime, timezone +import hashlib +import io +import json +import os +from pathlib import Path +import platform +import socket +import subprocess +import sys +from tempfile import TemporaryDirectory +import time +from uuid import uuid4 + +from PIL import Image + +from helpers import Client, ROOT +from verify_restart import verify as verify_restart + + +def main(): + with socket.socket() as sock: + sock.bind(('127.0.0.1', 0)) + port = sock.getsockname()[1] + project = 'lab5-check-' + uuid4().hex[:10] + base = 'http://127.0.0.1:' + str(port) + env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), BASE_URL=base, + GRAFANA_PORT='0', + API_REPLICAS='2', ENABLE_FAULT_INJECTION='1', JOB_TIMEOUT_SECONDS='5', + JOB_LEASE_SECONDS='15', JOB_MAX_ATTEMPTS='5', JOB_BACKOFF_SECONDS='0.15') + result = {'utc': datetime.now(timezone.utc).isoformat(), 'command': './scripts/check-docker.sh', + 'project': project, 'host': platform.system() + ' ' + platform.machine(), + 'source_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), + 'working_tree': True, 'configuration': {'concurrency': 2, 'prefetch': 1, 'attempts': 5, + 'timeout_seconds': 5, 'lease_seconds': 15, 'backoff_base_seconds': 0.15}, 'checks': {}} + + def command(args, timeout=240, check=True): + return subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, check=check) + + def compose(*args, timeout=240, check=True): + return command(['docker', 'compose', *args], timeout, check) + + def sql(query): + return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-c', query).stdout.strip() + + def worker_code(code, service='worker'): + return compose('exec', '-T', service, 'python', '-c', code).stdout.strip() + + def wait(predicate, seconds=60, description='condition'): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + value = predicate() + if value: + return value + time.sleep(0.2) + raise AssertionError('Timeout: ' + description) + + def fault(job_id, point, action='crash', service='worker'): + # docker cp позволяет установить failpoint также в остановленный publisher. + container = compose('ps', '-aq', service).stdout.strip() + with TemporaryDirectory(prefix='pipeline-fault-') as directory: + source = Path(directory) / 'fault' + source.write_text(action) + source.chmod(0o644) + command(['docker', 'cp', str(source), container + ':/tmp/pipeline-faults/' + job_id + '.' + point]) + + def hit(job_id, point, service='worker'): + probe = compose('exec', '-T', service, 'python', '-c', 'from pathlib import Path; print(Path(' + repr('/tmp/pipeline-faults/' + job_id + '.' + point + '.hit') + ').exists())', check=False) + return probe.returncode == 0 and probe.stdout.strip() == 'True' + + def job_id(att): + return sql("SELECT id FROM attachment_jobs WHERE attachment_id='" + att['id'] + "'") + + def objects(att): + prefix = 'attachments/' + att['id'].replace('-', '') + '/v1/' + return json.loads(worker_code('import json; from src import object_store as s; print(json.dumps(sorted(o.object_name for o in s.client.list_objects(s.BUCKET,prefix=' + repr(prefix) + ',recursive=True))))', service='cleanup')) + + def stage_exists(att): + key = sql("SELECT staging_key FROM attachment_jobs WHERE attachment_id='" + att['id'] + "'") + return int(worker_code('from src import object_store as s; print(sum(1 for o in s.client.list_objects(s.BUCKET,prefix=' + repr(key) + ',recursive=True)))', service='cleanup')) == 1 + + def record(name, **data): + result['checks'][name] = {'passed': True, **data} + print('PASS:', name, json.dumps(data, ensure_ascii=False), flush=True) + + def metadata(att): + return client.call('GET', '/api/v1/attachments/' + att['id'], actor=alice, schema='Attachment')['json'] + + def status(att, wanted, seconds=60): + return wait(lambda: (value if (value := metadata(att))['status'] == wanted else None), seconds, wanted) + + def upload(data=None, mime='image/png'): + return client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202, + actor=alice, raw=data or png, headers={'Content-Type': mime}, schema='Attachment')['json'] + + def assert_variants(att, original): + ready = status(att, 'ready') + assert {v['name'] for v in ready['variants']} == {'thumbnail', 'crop', 'optimized'} + original_response = client.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + assert original_response['bytes'] == original + assert ready['sha256'] == hashlib.sha256(original).hexdigest() + variants = {} + for variant in ready['variants']: + response = client.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + variant['name'], actor=bob) + data = response['bytes'] + assert len(data) == variant['size_bytes'] and hashlib.sha256(data).hexdigest() == variant['sha256'] + with Image.open(io.BytesIO(data)) as image: + image.load() + assert image.size == (variant['width'], variant['height']) + assert image.format == ('WEBP' if variant['name'] == 'optimized' else 'JPEG') + assert not image.getexif() + variants[variant['name']] = {'size': list(image.size), 'bytes': len(data)} + assert len(objects(att)) == 4 + return variants + + def queue_payload(att): + return json.loads(worker_code('''import json +from src import db,broker +from src.storage import utc +db.pool.open(wait=True) +with db.connection() as c: + j=c.execute("SELECT * FROM attachment_jobs WHERE attachment_id=%s",(''' + repr(att['id']) + ''',)).fetchone() + payload={'schema_version':1,'job_id':str(j['id']),'attachment_id':str(j['attachment_id']),'type':'attachment.process.v1','pipeline_version':1,'attempt':j['attempt'],'created_at':utc(j['created_at'])} +print(json.dumps(payload)) +db.pool.close() +''')) + + def duplicate(att, count=2): + payload = queue_payload(att) + from jsonschema import Draft202012Validator, FormatChecker + Draft202012Validator(json.loads((ROOT / 'contracts/jobs.schema.json').read_text()), format_checker=FormatChecker()).validate(payload) + worker_code('import json; from src import broker; c=broker.connect(); p=' + repr(json.dumps(payload)) + '; [c.xadd(broker.STREAM,{"job":p}) for _ in range(' + str(count) + ')]') + + try: + result['docker'] = command(['docker', 'info', '--format', '{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}']).stdout.strip() + command([str(ROOT / 'scripts/start.sh')], timeout=300) + for service in ('worker', 'publisher'): + worker_code("from pathlib import Path; Path('/tmp/pipeline-faults').mkdir(exist_ok=True)", service) + ids = compose('ps', '-aq').stdout.split() + infos = json.loads(command(['docker', 'inspect', *ids]).stdout) + result['containers'] = [{'id': info['Id'], 'service': info['Config']['Labels']['com.docker.compose.service'], + 'image': info['Config']['Image'], 'memory_bytes': info['HostConfig']['Memory'], + 'nano_cpus': info['HostConfig']['NanoCpus'], + 'mounts': [{'destination': m['Destination'], 'name': m.get('Name')} for m in info['Mounts']]} for info in infos] + assert sum(i['Config']['Labels']['com.docker.compose.service'] == 'api' for i in infos) == 2 + for info in infos: + assert info['HostConfig']['Memory'] and info['HostConfig']['NanoCpus'] + if info['Config']['Labels']['com.docker.compose.service'] not in {'lb', 'grafana'}: + assert not info['HostConfig']['PortBindings'] + record('topology_and_limits', api_replicas=2, broker='Valkey Streams with AOF fsync always', worker_processes=2) + public = command(['make', 'check', 'validate', 'test', 'PYTHON=' + sys.executable]) + print(public.stdout, end='', flush=True) + record('public_smoke_and_schema') + unit = command([sys.executable, '-m', 'unittest', 'discover', '-s', 'student-tests', '-v']) + print(unit.stdout + unit.stderr, end='', flush=True) + record('student_api_regressions') + client = Client(base) + alice, bob, eve, chat = client.bootstrap() + png = (ROOT / 'tests/fixtures/sample.png').read_bytes() + + # Worker stopped: queued already linkable; API and broker restarts preserve accepted bytes/job. + compose('stop', '--timeout', '3', 'worker') + att = upload() + assert att['status'] == 'queued' and stage_exists(att) + client.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=alice) + msg = client.message(bob, chat, '', attachments=[att['id']])['json'] + jid = job_id(att) + wait(lambda: sql("SELECT published_at IS NOT NULL FROM attachment_outbox WHERE job_id='" + jid + "'") == 't') + messages = json.loads(worker_code('from src import broker; import json; c=broker.connect(); print(json.dumps([json.loads(fields["job"]) for _,fields in c.xrange(broker.STREAM)]))', service='publisher')) + from jsonschema import Draft202012Validator, FormatChecker + validator = Draft202012Validator(json.loads((ROOT / 'contracts/jobs.schema.json').read_text()), format_checker=FormatChecker()) + assert messages + for payload in messages: + validator.validate(payload) + compose('restart', '--timeout', '3', 'api', 'broker') + command([str(ROOT / 'scripts/wait-ready.sh')]) + assert metadata(att)['status'] == 'queued' + compose('start', 'worker') + variants = assert_variants(att, png) + assert variants['thumbnail']['size'] == [256, 160] and variants['crop']['size'] == [128, 128] + record('worker_offline_api_broker_restart', attachment_id=att['id'], job_id=jid, message_id=msg['id'], variants=variants) + + # COMMIT -> publish gap and XADD -> outbox COMMIT gap, each crash is automatic. + for point in ('before_publish', 'after_publish'): + compose('stop', '--timeout', '3', 'publisher') + att = upload(); jid = job_id(att) + assert sql("SELECT published_at IS NULL FROM attachment_outbox WHERE job_id='" + jid + "'") == 't' + fault(jid, point, service='publisher') + compose('start', 'publisher') + wait(lambda: hit(jid, point, 'publisher'), description=point) + assert_variants(att, png) + record('outbox_crash_' + point, job_id=jid, automatic_recovery=True) + + # Worker crash after PUT before result COMMIT and after result COMMIT before ACK. + for point in ('after_objects', 'before_ack'): + compose('stop', '--timeout', '3', 'publisher') + att = upload(); jid = job_id(att) + fault(jid, point) + compose('start', 'publisher') + wait(lambda: hit(jid, point), description=point) + assert_variants(att, png) + if point == 'after_objects': + assert int(sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'")) >= 2 + snapshot = metadata(att) + duplicate(att, 4) + wait(lambda: worker_code('from src import broker; c=broker.connect(); print(c.xpending(broker.STREAM,broker.GROUP)["pending"])') == '0', seconds=45, description='pending ACK') + assert metadata(att) == snapshot and len(objects(att)) == 4 + record('worker_crash_' + point, job_id=jid, attempt=int(sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'")), canonical_objects=4, duplicate_noop=True) + + # Transient S3 errors, jittered backoff, exhausted registry and explicit replay. + compose('stop', '--timeout', '3', 'publisher') + att = upload(); jid = job_id(att) + fault(jid, 'before_process', 'transient:5') + compose('start', 'publisher') + status(att, 'failed') + failed = metadata(att) + assert failed['error_code'] == 'retry_exhausted' + client.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=alice) + client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=bob, + body={'text': '', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': uuid4().hex}) + registry = json.loads(compose('exec', '-T', 'worker', 'python', '-m', 'src.job_admin', 'failed').stdout) + assert any(row['job_id'] == jid for row in registry) + compose('exec', '-T', 'worker', 'python', '-m', 'src.job_admin', 'replay', jid) + assert_variants(att, png) + attempt = sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") + rejected = compose('exec', '-T', 'worker', 'python', '-m', 'src.job_admin', 'replay', jid, check=False) + assert rejected.returncode != 0 + assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == attempt + audit = json.loads(sql("SELECT json_agg(json_build_object('event',event,'attempt',attempt,'error_code',error_code,'at',created_at) ORDER BY id) FROM attachment_job_events WHERE job_id='" + jid + "'")) + record('bounded_retry_failed_registry_replay', job_id=jid, max_automatic_attempts=5, audit=audit, ready_replay_rejected=True) + + # Permanent error from damaged staging: one attempt, safe code, no retry. + compose('stop', '--timeout', '3', 'publisher') + invalid = upload(); jid = job_id(invalid) + key = sql("SELECT staging_key FROM attachment_jobs WHERE id='" + jid + "'") + worker_code('import io; from src import object_store as s; s.put(' + repr(key) + ',io.BytesIO(b"bad"),3,"image/png")') + compose('start', 'publisher') + status(invalid, 'failed') + assert metadata(invalid)['error_code'] == 'invalid_image' + assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == '1' + record('permanent_error', job_id=jid, attempt=1, error_code='invalid_image') + + # Реальные timeout и MemoryError внутри ограниченного resize subprocess. + for action, error_code in [('timeout', 'processing_timeout'), ('memory', 'resource_limit')]: + compose('stop', '--timeout', '3', 'publisher') + att = upload(); jid = job_id(att) + fault(jid, 'transform', action) + compose('start', 'publisher') + assert_variants(att, png) + assert sql("SELECT count(*) FROM attachment_job_events WHERE job_id='" + jid + "' AND error_code='" + error_code + "'") == '1' + record('image_subprocess_' + action, job_id=jid, timeout_seconds=5, task_memory_mb=640, recovered=True) + + # Alpha, small input, EXIF rotation and floor rounding in real HTTP downloads. + cases = [] + alpha = io.BytesIO(); Image.new('RGBA', (17, 9), (255, 0, 0, 0)).save(alpha, format='PNG') + att = upload(alpha.getvalue()) + data = assert_variants(att, alpha.getvalue()) + assert data['thumbnail']['size'] == [17, 9] and data['optimized']['size'] == [17, 9] + thumb = client.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes'] + with Image.open(io.BytesIO(thumb)) as image: + assert all(channel >= 250 for channel in image.getpixel((8, 4))) + cases.append({'case': 'small_alpha_on_white', 'variants': data}) + photo = io.BytesIO(); exif = Image.Exif(); exif[274] = 6 + Image.new('RGB', (321, 201), 'orange').save(photo, format='JPEG', exif=exif) + att = upload(photo.getvalue(), 'image/jpeg') + data = assert_variants(att, photo.getvalue()) + assert data['thumbnail']['size'] == [160, 256] + cases.append({'case': 'exif_orientation_6_and_floor', 'variants': data}) + record('image_semantics', cases=cases) + + # Cleanup preserves active staging even if old; removes ready and expired failed. + compose('stop', '--timeout', '3', 'worker') + active = upload() + sql("UPDATE attachments SET created_at=clock_timestamp()-interval '2 days',updated_at=clock_timestamp()-interval '2 days' WHERE id='" + active['id'] + "'") + sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '2 days' WHERE id='" + invalid['id'] + "'") + orphan_id = str(uuid4()) + orphan_key = 'attachments/staging/' + orphan_id.replace('-', '') + sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + orphan_id + "','" + orphan_key + "',clock_timestamp()-interval '2 days')") + worker_code('import io; from src import object_store as s; s.put(' + repr(orphan_key) + ',io.BytesIO(b"orphan"),6,"image/png")', service='cleanup') + compose('run', '--rm', '--no-deps', 'cleanup', 'python', '-m', 'src.cleanup', '--once') + assert sql("SELECT count(*) FROM attachment_uploads WHERE id='" + orphan_id + "'") == '0' + assert worker_code('from src import object_store as s; print(sum(1 for o in s.client.list_objects(s.BUCKET,prefix=' + repr(orphan_key) + ',recursive=True)))', service='cleanup') == '0' + + assert stage_exists(active) + assert not stage_exists(invalid) + assert not stage_exists(att) + rejected = compose('exec', '-T', 'cleanup', 'python', '-m', 'src.job_admin', 'replay', job_id(invalid), check=False) + assert rejected.returncode != 0 + compose('start', 'worker'); assert_variants(active, png) + record('staging_cleanup', active_preserved=True, orphan_removed=True, ready_removed=True, expired_failed_removed=True, expired_replay_rejected=True) + + # Broker outage doesn't force synchronous resize: outbox recovers after restart. + compose('stop', '--timeout', '3', 'broker') + att = upload() + assert att['status'] == 'queued' + client.message(alice, chat, 'Broker offline; text API still works') + compose('start', 'broker'); assert_variants(att, png) + record('broker_offline_outbox', attachment_id=att['id'], recovered=True) + + record('public_persistence', restart_proof=verify_restart(env)) + failover = command(['make', 'test-failover', 'PYTHON=' + sys.executable, 'ACTION_SCRIPT=scripts/stop-one.sh']) + print(failover.stdout, end='', flush=True) + compose('start', 'api') + command([str(ROOT / 'scripts/wait-ready.sh')]) + record('public_api_failover') + result['passed'] = True + print('PASS: all lab5 grade-4 criteria completed', flush=True) + except subprocess.CalledProcessError as error: + # Команды не содержат credentials, но runtime logs не сохраняем автоматически. + print('Command failed:', error.cmd[:4], error.returncode, flush=True) + print(error.stdout, error.stderr, flush=True) + raise + finally: + output = ROOT / os.environ.get('LAB5_EVIDENCE_FILE', 'evidence/lab5-checks.json') + output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n') + if os.environ.get('KEEP_TEST_STACK') == '1' and not result.get('passed'): + print('Kept isolated failed test stack:', project, flush=True) + else: + compose('down', '--volumes', '--timeout', '5') + print('Removed isolated test stack:', project, flush=True) + + +if __name__ == '__main__': + main() diff --git a/student-tests/lab6_observability.py b/student-tests/lab6_observability.py new file mode 100644 index 0000000..7455da9 --- /dev/null +++ b/student-tests/lab6_observability.py @@ -0,0 +1,117 @@ +"""Live checks: stored signals, retry causality, privacy and bounded backend outage.""" +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import subprocess +import sys +from tempfile import TemporaryDirectory +import time +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(ROOT / 'tests'), str(ROOT)] +from client import Client +from research.observations import check_dashboard, compose, get, logs, query, scalar, save, spans, sql, trace, wait + + +def main(): + started = time.time() + result = {'utc': datetime.now(timezone.utc).isoformat(), 'checks': {}} + client = Client() + alice, bob, _, chat = client.bootstrap() + canary = 'PRIVATE-BODY-' + uuid4().hex + client.message(alice, chat, canary) + png = (ROOT / 'tests/fixtures/sample.png').read_bytes() + result['dashboard'] = get('/api/dashboards/uid/messenger-lab6')['dashboard']['uid'] + assert {s['uid'] for s in get('/api/datasources')} == {'prometheus', 'loki', 'tempo'} + compose('exec', '-T', 'worker', 'python', '-c', "from pathlib import Path; Path('/tmp/pipeline-faults').mkdir(exist_ok=True)") + # Stop before accepting so a one-shot worker failure is installed deterministically. + compose('stop', '--timeout', '5', 'worker') + try: + container = compose('ps', '-aq', 'worker').stdout.strip() + response = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + canary + '.png', 202, + actor=alice, raw=png, headers={'Content-Type': 'image/png'}, schema='Attachment') + att = response['json']; trace_id = response['headers']['X-Trace-Id'] + job_id = sql("SELECT id FROM attachment_jobs WHERE attachment_id='" + att['id'] + "'") + with TemporaryDirectory() as directory: + marker = Path(directory) / 'fault'; marker.write_text('transient'); marker.chmod(0o644) + subprocess.run(['docker', 'cp', str(marker), container + ':/tmp/pipeline-faults/' + job_id + '.before_process'], check=True, capture_output=True) + compose('start', 'worker') + ready = client.wait_ready(alice, att) + assert ready['status'] == 'ready' + wait(lambda: len(spans(trace(trace_id))) >= 10) + time.sleep(5) + stored = trace(trace_id) + chain = spans(stored) + names = [s['name'] for s in chain] + assert {'messenger-api', 'messenger-publisher', 'messenger-worker'} <= {s['service'] for s in chain} + assert names.count('attachments publish') >= 2 and names.count('attachments process') >= 2 + assert 's3.put_object' in names and 's3.get_object' in names and 'postgresql.UPDATE' in names + identifiers = {s['spanId'] for s in chain} + assert all(not s.get('parentSpanId') or s['parentSpanId'] in identifiers for s in chain), 'Broken parent chain' + correlated = wait(lambda: (v if (v := logs('{service_name=~"messenger-api|messenger-worker"} |= "' + trace_id + '"'))['data']['result'] else None)) + raw = json.dumps([stored, correlated]) + assert canary not in raw and alice['token'] not in raw and alice['registration']['password'] not in raw + save('lab6-trace-retry.json', stored); save('lab6-correlated-logs.json', correlated) + result['checks']['retry_trace_and_logs'] = {'passed': True, 'trace_id': trace_id, + 'request_id': response['headers']['X-Request-Id'], 'job_id': job_id, 'span_count': len(chain)} + finally: + compose('start', 'worker') + + wait(lambda: query('messenger_http_requests_total')) + for name in ['messenger_http_duration_seconds_bucket','messenger_job_attempts_total', + 'messenger_job_duration_seconds_bucket','messenger_queue_depth','messenger_queue_oldest_age_seconds', + 'messenger_db_pool','messenger_worker_busy','messenger_worker_capacity', + 'messenger_process_memory_bytes','messenger_process_cpu_time_seconds_total', + 'messenger_attachment_ready_duration_seconds_bucket','messenger_ready_deadlines_total']: + assert query(name), 'Missing metric: ' + name + series = query('{__name__=~"messenger_.*"}') + forbidden = {'user_id','chat_id','message_id','attachment_id','request_id','filename','url','http_target'} + assert all(not forbidden.intersection(s['metric']) for s in series) + result['checks']['metrics_and_cardinality'] = {'passed': True, 'series': len(series)} + + # Count actual missing stored log records, not just retry attempts. + lost_candidates = [] + before = query('otelcol_exporter_send_failed_log_records_total') + stats = [] + compose('stop', '--timeout', '5', 'loki') + outage_start = time.time() + try: + for i in range(24): + response = client.message(alice, chat, 'synthetic backend outage') + lost_candidates.append(response['headers']['X-Request-Id']) + if i % 6 == 0: + stats.append(subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}', + compose('ps','-q','otel-collector').stdout.strip()],text=True).strip()) + time.sleep(1.5) + finally: + compose('start', 'loki') + wait(lambda: logs('{service_name="messenger-api"}')['status'] == 'success') + time.sleep(20) + retrieved = logs('{service_name="messenger-api"} | json | event="http.request"', start=outage_start-1, limit=5000) + encoded = json.dumps(retrieved) + missing = [rid for rid in lost_candidates if rid not in encoded] + collector_id = compose('ps','-q','otel-collector').stdout.strip() + state = json.loads(subprocess.check_output(['docker','inspect',collector_id],text=True))[0] + assert state['State']['Running'] and not state['State']['OOMKilled'] + assert state['HostConfig']['Memory'] == 256*1024*1024 + assert missing, 'Expected finite retry timeout to cause measured telemetry loss' + client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=alice, schema='MessagePage') + result['checks']['backend_outage'] = {'passed': True,'backend':'loki', 'successful_writes':24, + 'outage_seconds':round(time.time()-outage_start-20,2),'missing_request_logs':len(missing), + 'tested_request_ids':lost_candidates, 'missing_request_ids':missing, 'collector_memory_samples':stats, + 'collector_limit_bytes':state['HostConfig']['Memory'], 'oom_killed':False, + 'failed_exports_before':before,'failed_exports_after':query('otelcol_exporter_send_failed_log_records_total')} + # Every Prometheus expression referenced by the provisioned dashboard parses and executes. + dashboard_check = check_dashboard() + save('lab6-dashboard-checks.json', dashboard_check) + result['checks']['provisioned_dashboard_queries'] = {'passed':True,'panels':len(dashboard_check['panels'])} + result['passed'] = True + result['duration_seconds'] = round(time.time()-started,2) + save('lab6-checks.json',result) + print(json.dumps(result,ensure_ascii=False,indent=2)) + + +if __name__ == '__main__': + main() diff --git a/student-tests/postgres_pause.py b/student-tests/postgres_pause.py new file mode 100644 index 0000000..07bd2fb --- /dev/null +++ b/student-tests/postgres_pause.py @@ -0,0 +1,145 @@ +"""Пауза живой PostgreSQL: прямой readiness, ограниченное SQL I/O и восстановление.""" +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +from uuid import uuid4 + +from helpers import Client, ROOT +from client import Failure + + +def check_postgres_pause(compose, apis, base): + client = Client(base) + alice, bob, _, chat = client.bootstrap() + saved_message = client.message(alice, chat, "До паузы PostgreSQL")["json"] + before = {info["Id"]:info["State"]["StartedAt"] for info in apis() if info["State"]["Running"]} + probe_id = sorted(before)[0] + + def direct_probe(path, expected): + # HTTP напрямую в API, без HAProxy и его собственных таймаутов/503. + code = """ +import json,time +from urllib.error import HTTPError +from urllib.request import urlopen +started=time.monotonic() +try: response=urlopen('http://127.0.0.1:8080/PATH',timeout=5) +except HTTPError as error: response=error +with response: + body=json.load(response) + elapsed=time.monotonic()-started + assert response.status==EXPECTED, response.status + assert response.headers['X-Request-Id'] and response.headers['X-Instance-Id'] + if response.status==503: + assert body['error']['code']=='unavailable' + assert body['error']['request_id']==response.headers['X-Request-Id'] + assert elapsed<4.5, elapsed + print(json.dumps({'status':response.status,'elapsed_seconds':round(elapsed,3)})) +""".replace("PATH", path).replace("EXPECTED", str(expected)) + completed = subprocess.run(["docker", "exec", probe_id, "python", "-c", code], + capture_output=True, text=True, timeout=8, check=True) + return json.loads(completed.stdout) + + # Соединение получено ДО pause: проверяем именно выполняющийся SQL, а не + # только timeout получения нового соединения или check callback пула. + query_code = """ +import json,sys,time +import psycopg +from src import db +db.pool.open(wait=True,timeout=10) +conn=db.pool.getconn() +conn.execute('SET statement_timeout=0') +print('connected',flush=True) +sys.stdin.readline() +started=time.monotonic() +try: + try: conn.execute('SELECT 1') + except psycopg.OperationalError: + elapsed=time.monotonic()-started + assert elapsed<4.5, elapsed + assert conn.closed, 'timed out connection was not discarded' + print(json.dumps({'elapsed_seconds':round(elapsed,3),'connection_closed':conn.closed}),flush=True) + else: raise AssertionError('paused database answered a query') +finally: + db.pool.putconn(conn) + db.pool.close() +""" + query = subprocess.Popen(["docker", "exec", "-i", probe_id, "python", "-u", "-c", query_code], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = {} + try: + assert query.stdout.readline().strip() == "connected" + compose("pause", "postgres") + paused_at = time.monotonic() + output, error = query.communicate("\n", timeout=10) + assert query.returncode == 0, "SQL I/O deadline failed: " + error + result["sql_without_server_timeout"] = json.loads(output) + result["direct_ready"] = direct_probe("health/ready", 503) + result["direct_live"] = direct_probe("health/live", 200) + # Повторные запросы при длительной паузе не должны оставлять занятые потоки. + with ThreadPoolExecutor(max_workers=10) as executor: + result["repeated_ready"] = list(executor.map(lambda _: direct_probe("health/ready", 503), range(10))) + # Сервер остаётся на pause не менее 12 секунд; ограничиваем именно API. + remaining = 12 - (time.monotonic() - paused_at) + if remaining > 0: + time.sleep(remaining) + result["ready_after_12_seconds"] = direct_probe("health/ready", 503) + result["live_after_repeated_requests"] = direct_probe("health/live", 200) + result["paused_seconds"] = round(time.monotonic() - paused_at, 3) + finally: + compose("unpause", "postgres") + if query.poll() is None: + query.communicate(timeout=10) + + resumed_at = time.monotonic() + while True: + try: + result["recovered_ready"] = direct_probe("health/ready", 200) + client.call("GET", "/api/v1/users/me", actor=alice, schema="User") + messages = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage")["json"]["items"] + assert saved_message in messages + break + except (subprocess.CalledProcessError, Failure): + assert time.monotonic() - resumed_at < 30, "readiness did not recover after unpause" + time.sleep(.2) + assert before == {info["Id"]:info["State"]["StartedAt"] for info in apis() if info["State"]["Running"]} + result.update({"recovery_seconds":round(time.monotonic()-resumed_at,3), "api_not_restarted":True, + "existing_session_and_message_preserved":True, "passed":True}) + print("PASS: paused PostgreSQL: SQL and direct readiness bounded; live works; unpause restores API without restart", flush=True) + return result + + +def main(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + project = "lab4-pg-pause-" + uuid4().hex[:10] + base = f"http://127.0.0.1:{port}" + env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), API_REPLICAS="2", BASE_URL=base) + + def compose(*args): + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, env=env, text=True, timeout=180).strip() + + def apis(): + ids = compose("ps", "-a", "-q", "api").split() + return json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True)) + + result = {"utc":datetime.now(timezone.utc).isoformat(), "project":project, + "command":".venv/bin/python student-tests/postgres_pause.py", "passed":False} + try: + compose("up", "-d", "--no-build", "lb") + subprocess.run([str(ROOT / "scripts/wait-ready.sh")], cwd=ROOT, env=env, check=True, timeout=120) + result.update(check_postgres_pause(compose, apis, base)) + print(json.dumps(result, ensure_ascii=False), flush=True) + finally: + Path(ROOT / "evidence/postgres-pause.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + compose("down", "--volumes", "--timeout", "8") + + +if __name__ == "__main__": + main() diff --git a/student-tests/promtool-slo.yaml b/student-tests/promtool-slo.yaml new file mode 100644 index 0000000..a418ffc --- /dev/null +++ b/student-tests/promtool-slo.yaml @@ -0,0 +1,199 @@ +rule_files: +- ../infra/otel/alerts.yaml +evaluation_interval: 5s +tests: +- name: 'isolated spike: single-window fires, multiwindow does not' + interval: 5s + input_series: + - series: messenger_http_requests_total{status_class="5xx"} + values: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 10 10 10 10 10 10 + 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 + 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 + 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 + 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 + 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 + 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 + - series: messenger_http_requests_total{status_class="2xx"} + values: 0.0 12.5 25.0 37.5 50.0 62.5 75.0 87.5 100.0 112.5 125.0 137.5 150.0 162.5 + 175.0 187.5 200.0 212.5 225.0 237.5 250.0 262.5 275.0 287.5 300.0 312.5 325.0 + 337.5 350.0 362.5 375.0 387.5 400.0 412.5 425.0 437.5 450.0 462.5 475.0 487.5 + 500.0 512.5 525.0 537.5 550.0 562.5 575.0 587.5 600.0 612.5 625.0 637.5 650.0 + 662.5 675.0 687.5 700.0 712.5 725.0 737.5 750.0 762.5 775.0 787.5 800.0 812.5 + 825.0 837.5 850.0 862.5 875.0 887.5 900.0 912.5 925.0 937.5 950.0 962.5 975.0 + 987.5 1000.0 1012.5 1025.0 1037.5 1050.0 1062.5 1075.0 1087.5 1100.0 1112.5 + 1125.0 1137.5 1150.0 1162.5 1175.0 1187.5 1200.0 1212.5 1225.0 1237.5 1250.0 + 1262.5 1275.0 1287.5 1300.0 1312.5 1325.0 1337.5 1350.0 1362.5 1375.0 1387.5 + 1400.0 1412.5 1425.0 1437.5 1450.0 1462.5 1475.0 1487.5 1500.0 1512.5 1525.0 + 1537.5 1550.0 1562.5 1575.0 1587.5 1600.0 1612.5 1625.0 1637.5 1650.0 1662.5 + 1675.0 1687.5 1700.0 1712.5 1725.0 1737.5 1750.0 1762.5 1775.0 1787.5 1800.0 + 1812.5 1825.0 1837.5 1850.0 1862.5 1875.0 1887.5 1900.0 1912.5 1925.0 1937.5 + 1950.0 1962.5 1975.0 1987.5 2000.0 2012.5 2025.0 2037.5 2050.0 2062.5 2075.0 + 2087.5 2100.0 2112.5 2125.0 2137.5 2150.0 2162.5 2175.0 2187.5 2200.0 2212.5 + 2225.0 2237.5 2250.0 + alert_rule_test: + - eval_time: 140s + alertname: APIHighErrorRate + exp_alerts: + - exp_labels: + severity: warning + exp_annotations: + summary: API 5xx > 5% for 30s (single-window comparator) + runbook: docs/runbook.md#api + runbook_url: https://gitea.privcloud.tech/Savelev_Ivan/mtusi2026lab6/src/branch/main/docs/runbook.md#api + - eval_time: 140s + alertname: APIAvailabilityFastBurn + exp_alerts: [] + - eval_time: 140s + alertname: APIAvailabilitySlowBurn + exp_alerts: [] + - eval_time: 5m + alertname: APIAvailabilityFastBurn + exp_alerts: [] +- name: sustained errors and recovery + interval: 5s + input_series: + - series: messenger_http_requests_total{status_class="5xx"} + values: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 10 20 30 40 50 60 70 + 80 90 100 110 120 130 140 150 160 170 180 190 200 210 220 230 240 240 240 240 + 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 + 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 + 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 + 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 + 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 + 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 + 240 240 240 240 240 240 240 240 240 240 240 240 240 240 240 + - series: messenger_http_requests_total{status_class="2xx"} + values: 0.0 12.5 25.0 37.5 50.0 62.5 75.0 87.5 100.0 112.5 125.0 137.5 150.0 162.5 + 175.0 187.5 200.0 212.5 225.0 237.5 250.0 262.5 275.0 287.5 300.0 302.5 305.0 + 307.5 310.0 312.5 315.0 317.5 320.0 322.5 325.0 327.5 330.0 332.5 335.0 337.5 + 340.0 342.5 345.0 347.5 350.0 352.5 355.0 357.5 360.0 372.5 385.0 397.5 410.0 + 422.5 435.0 447.5 460.0 472.5 485.0 497.5 510.0 522.5 535.0 547.5 560.0 572.5 + 585.0 597.5 610.0 622.5 635.0 647.5 660.0 672.5 685.0 697.5 710.0 722.5 735.0 + 747.5 760.0 772.5 785.0 797.5 810.0 822.5 835.0 847.5 860.0 872.5 885.0 897.5 + 910.0 922.5 935.0 947.5 960.0 972.5 985.0 997.5 1010.0 1022.5 1035.0 1047.5 + 1060.0 1072.5 1085.0 1097.5 1110.0 1122.5 1135.0 1147.5 1160.0 1172.5 1185.0 + 1197.5 1210.0 1222.5 1235.0 1247.5 1260.0 1272.5 1285.0 1297.5 1310.0 1322.5 + 1335.0 1347.5 1360.0 1372.5 1385.0 1397.5 1410.0 1422.5 1435.0 1447.5 1460.0 + 1472.5 1485.0 1497.5 1510.0 1522.5 1535.0 1547.5 1560.0 1572.5 1585.0 1597.5 + 1610.0 1622.5 1635.0 1647.5 1660.0 1672.5 1685.0 1697.5 1710.0 1722.5 1735.0 + 1747.5 1760.0 1772.5 1785.0 1797.5 1810.0 1822.5 1835.0 1847.5 1860.0 1872.5 + 1885.0 1897.5 1910.0 1922.5 1935.0 1947.5 1960.0 1972.5 1985.0 1997.5 2010.0 + alert_rule_test: + - eval_time: 210s + alertname: APIAvailabilityFastBurn + exp_alerts: + - exp_labels: + severity: critical + slo: api + exp_annotations: + summary: 'api: 6x budget burn in both 2m and 30s' + description: Budget 0.01; rolling SLO window 1h. Requires real traffic, not + health requests. + runbook_url: https://gitea.privcloud.tech/Savelev_Ivan/mtusi2026lab6/src/branch/main/docs/runbook.md#api + runbook: docs/runbook.md#api + - eval_time: 210s + alertname: APIAvailabilitySlowBurn + exp_alerts: + - exp_labels: + severity: warning + slo: api + exp_annotations: + summary: 'api: 3x budget burn in both 6m and 1m' + description: Budget 0.01; rolling SLO window 1h. Requires real traffic, not + health requests. + runbook_url: https://gitea.privcloud.tech/Savelev_Ivan/mtusi2026lab6/src/branch/main/docs/runbook.md#api + runbook: docs/runbook.md#api + - eval_time: 7m + alertname: APIAvailabilityFastBurn + exp_alerts: [] + - eval_time: 7m + alertname: APIAvailabilitySlowBurn + exp_alerts: [] +- name: stalled jobs count as bad and recover + interval: 5s + input_series: + - series: messenger_ready_deadlines_total{outcome="bad"} + values: 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.5 5.0 7.5 10.0 12.5 15.0 17.5 20.0 22.5 25.0 27.5 + 30.0 32.5 35.0 37.5 40.0 42.5 45.0 47.5 50.0 52.5 55.0 57.5 60.0 60.0 60.0 60.0 + 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 + 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 + 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 + 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 + 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 + 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 + 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 + 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 + 60.0 + - series: messenger_ready_deadlines_total{outcome="good"} + values: 0.0 2.5 5.0 7.5 10.0 12.5 15.0 17.5 20.0 22.5 25.0 27.5 30.0 32.5 35.0 + 37.5 40.0 42.5 45.0 47.5 50.0 52.5 55.0 57.5 60.0 60.0 60.0 60.0 60.0 60.0 60.0 + 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 60.0 + 60.0 60.0 62.5 65.0 67.5 70.0 72.5 75.0 77.5 80.0 82.5 85.0 87.5 90.0 92.5 95.0 + 97.5 100.0 102.5 105.0 107.5 110.0 112.5 115.0 117.5 120.0 122.5 125.0 127.5 + 130.0 132.5 135.0 137.5 140.0 142.5 145.0 147.5 150.0 152.5 155.0 157.5 160.0 + 162.5 165.0 167.5 170.0 172.5 175.0 177.5 180.0 182.5 185.0 187.5 190.0 192.5 + 195.0 197.5 200.0 202.5 205.0 207.5 210.0 212.5 215.0 217.5 220.0 222.5 225.0 + 227.5 230.0 232.5 235.0 237.5 240.0 242.5 245.0 247.5 250.0 252.5 255.0 257.5 + 260.0 262.5 265.0 267.5 270.0 272.5 275.0 277.5 280.0 282.5 285.0 287.5 290.0 + 292.5 295.0 297.5 300.0 302.5 305.0 307.5 310.0 312.5 315.0 317.5 320.0 322.5 + 325.0 327.5 330.0 332.5 335.0 337.5 340.0 342.5 345.0 347.5 350.0 352.5 355.0 + 357.5 360.0 362.5 365.0 367.5 370.0 372.5 375.0 377.5 380.0 382.5 385.0 387.5 + 390.0 + alert_rule_test: + - eval_time: 210s + alertname: AttachmentReadyFastBurn + exp_alerts: + - exp_labels: + severity: critical + slo: ready + exp_annotations: + summary: 'ready: 6x budget burn in both 2m and 30s' + description: Budget 0.05; rolling SLO window 1h. Requires real traffic, not + health requests. + runbook_url: https://gitea.privcloud.tech/Savelev_Ivan/mtusi2026lab6/src/branch/main/docs/runbook.md#ready + runbook: docs/runbook.md#ready + - eval_time: 210s + alertname: AttachmentReadySlowBurn + exp_alerts: + - exp_labels: + severity: warning + slo: ready + exp_annotations: + summary: 'ready: 3x budget burn in both 6m and 1m' + description: Budget 0.05; rolling SLO window 1h. Requires real traffic, not + health requests. + runbook_url: https://gitea.privcloud.tech/Savelev_Ivan/mtusi2026lab6/src/branch/main/docs/runbook.md#ready + runbook: docs/runbook.md#ready + - eval_time: 7m + alertname: AttachmentReadyFastBurn + exp_alerts: [] + - eval_time: 7m + alertname: AttachmentReadySlowBurn + exp_alerts: [] +- name: no traffic is not a passing SLO or an error burn + interval: 5s + input_series: + - series: messenger_http_requests_total{status_class="5xx"} + values: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + - series: messenger_http_requests_total{status_class="2xx"} + values: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + alert_rule_test: + - eval_time: 5m + alertname: APIAvailabilityFastBurn + exp_alerts: [] + - eval_time: 5m + alertname: APIAvailabilitySlowBurn + exp_alerts: [] + promql_expr_test: + - eval_time: 5m + expr: lab6:api_bad_ratio:2m == 0 + exp_samples: [] diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 0000000..08940ae --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,164 @@ +import base64 +import json +from urllib.parse import urlencode +from uuid import uuid4 + +from helpers import ApiTestCase + + +class ApiTests(ApiTestCase): + def test_duplicate_username(self): + self.assert_error("POST", "/api/v1/users", 409, "conflict", body=self.alice["registration"]) + + def test_invalid_user_fields(self): + invalid_bodies = [ + {}, [], None, + {"username": "ab", "display_name": "Имя"}, + {"username": "UPPER", "display_name": "Имя"}, + {"username": "name\n", "display_name": "Имя"}, + {"username": "язык", "display_name": "Имя"}, + {"username": "a" * 33, "display_name": "Имя"}, + {"username": 123, "display_name": "Имя"}, + {"username": "valid_name", "display_name": " \t\n"}, + {"username": "valid_name", "display_name": "👋" * 101}, + {"username": "valid_name", "display_name": False}, + {"username": "valid_name", "display_name": "Имя", "extra": 1}, + ] + for body in invalid_bodies: + if isinstance(body, dict): + body = {**body, "password": "Synthetic-password-123"} + with self.subTest(body=body): + self.assert_error("POST", "/api/v1/users", 400, "invalid_request", + raw=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + + def test_malformed_json_and_encoding(self): + invalid_bodies = [ + b'{"username":', b'', b'not JSON', b'\xff', + '{"username":"valid_name","display_name":"Имя"}'.encode("utf-16"), + b'{"username":"valid_name","display_name":"\\ud800"}', + ] + for body in invalid_bodies: + with self.subTest(body=body): + self.assert_error("POST", "/api/v1/users", 400, "invalid_request", + raw=body, headers={"Content-Type": "application/json"}) + + def test_wrong_content_type(self): + self.assert_error("POST", "/api/v1/users", 415, "unsupported_media_type", + raw=b'{}', headers={"Content-Type": "text/plain"}) + + def test_identification_and_authentication_order(self): + self.assert_error("GET", "/api/v1/users/me", 401, "unauthorized") + self.assert_error("GET", "/api/v1/users/me", 401, "unauthorized", + headers={"X-User-Id": self.alice["id"]}) + self.assert_error("GET", "/api/v1/users/me", 401, "unauthorized", + headers={"Authorization": "Bearer unknown-session"}) + self.assert_error("POST", self.path + "/messages", 401, "unauthorized", + raw=b'{', headers={"Content-Type": "application/json"}) + me = self.client.call("GET", "/api/v1/users/me", schema="User", actor=self.alice, + headers={"X-User-Id": self.bob["id"]})["json"] + self.assertEqual(me, self.alice["user"]) + + def test_invalid_chat_members(self): + invalid_members = [ + [], [self.bob["id"]], [self.alice["id"], str(uuid4())], + [self.alice["id"], self.alice["id"]], + [self.alice["id"], self.alice["id"].upper()], + [self.alice["id"], "not-a-uuid"], [True], self.alice["id"], + [self.alice["id"]] + [str(uuid4()) for _ in range(100)], + ] + for members in invalid_members: + with self.subTest(members=members): + self.assert_error("POST", "/api/v1/chats", 400, "invalid_request", actor=self.alice, + body={"title": "Чат", "member_ids": members}) + chats = self.read_all("/api/v1/chats", self.alice, "ChatPage") + self.assertEqual([chat["id"] for chat in chats], [self.chat["id"]]) + + def test_invalid_chat_title(self): + for title in ("", " \n", "a" * 101, 123, None): + with self.subTest(title=title): + self.assert_error("POST", "/api/v1/chats", 400, "invalid_request", actor=self.alice, + body={"title": title, "member_ids": [self.alice["id"]]}) + + def test_chat_access(self): + outsider = self.assert_error("GET", self.path, 404, "not_found", actor=self.eve) + missing = self.assert_error("GET", "/api/v1/chats/" + str(uuid4()), 404, "not_found", actor=self.eve) + self.assertEqual(outsider["message"], missing["message"]) + self.assert_error("GET", self.path + "/messages", 404, "not_found", actor=self.eve) + self.assert_error("POST", self.path + "/messages", 404, "not_found", actor=self.eve, body={"text": "Чужое"}) + self.assertEqual(self.read_all("/api/v1/chats", self.eve, "ChatPage"), []) + self.assert_error("GET", "/api/v1/chats/not-a-uuid", 400, "invalid_request", actor=self.alice) + + def test_invalid_message_fields(self): + for text in ("", " \t\n", "👋" * 4001, 123, True, None, []): + with self.subTest(text_type=type(text).__name__): + self.assert_error("POST", self.path + "/messages", 400, "invalid_request", + actor=self.alice, body={"text": text}) + self.assert_error("POST", self.path + "/messages", 400, "invalid_request", actor=self.alice, + body={"text": "Подмена автора", "sender_id": self.bob["id"]}) + self.assertEqual(self.read_all(self.path + "/messages", self.alice, "MessagePage"), []) + + def test_unicode_and_text_preservation(self): + for text in (" Привет, мир 👋\n", "👋" * 4000): + message = self.client.message(self.bob, self.chat, text)["json"] + self.assertEqual(message["text"], text) + self.assertEqual(message["sender_id"], self.bob["id"]) + items = self.read_all(self.path + "/messages", self.alice, "MessagePage") + self.assertEqual([item["text"] for item in items], [" Привет, мир 👋\n", "👋" * 4000]) + + def test_message_pagination_and_changing_limit(self): + sent = [self.client.message(self.alice, self.chat, f"Сообщение {i}")["json"] for i in range(8)] + items = self.read_all(self.path + "/messages", self.bob, "MessagePage", limit=3) + self.assertEqual(items, sorted(sent, key=lambda item: (item["created_at"], item["id"]))) + first = self.client.call("GET", self.path + "/messages?limit=1", actor=self.alice, schema="MessagePage")["json"] + query = urlencode({"limit": 100, "cursor": first["next_cursor"]}) + rest = self.client.call("GET", self.path + "/messages?" + query, actor=self.alice, schema="MessagePage")["json"] + self.assertEqual(first["items"] + rest["items"], items) + self.assertIsNone(rest["next_cursor"]) + + def test_default_page_size(self): + for i in range(51): + self.client.message(self.alice, self.chat, f"Сообщение {i}") + page = self.client.call("GET", self.path + "/messages", actor=self.alice, schema="MessagePage")["json"] + self.assertEqual(len(page["items"]), 50) + self.assertIsNotNone(page["next_cursor"]) + + def test_chat_pagination_and_cursor_owner(self): + for i in range(4): + self.client.call("POST", "/api/v1/chats", 201, actor=self.alice, + body={"title": f"Чат {i}", "member_ids": [self.alice["id"], self.bob["id"]]}) + chats = self.read_all("/api/v1/chats", self.alice, "ChatPage", limit=2) + self.assertEqual(len(chats), 5) + self.assertEqual(len({chat["id"] for chat in chats}), 5) + self.assertEqual(chats, sorted(chats, key=lambda item: (item["created_at"], item["id"]))) + first = self.client.call("GET", "/api/v1/chats?limit=1", actor=self.alice, schema="ChatPage")["json"] + query = urlencode({"cursor": first["next_cursor"]}) + self.assert_error("GET", "/api/v1/chats?" + query, 400, "invalid_request", actor=self.bob) + + def test_invalid_limits_and_cursors(self): + bad_queries = [{"limit": value} for value in ("0", "101", "-1", "1.5", "1.0", "abc", "")] + bad_queries += [{"cursor": value} for value in ("", "not-base64!", "e30=", "W10=")] + for path in ("/api/v1/chats", self.path + "/messages"): + for query in bad_queries: + with self.subTest(path=path, query=query): + self.assert_error("GET", path + "?" + urlencode(query), 400, "invalid_request", actor=self.alice) + + def test_cursor_cannot_be_used_for_another_chat_or_changed(self): + for i in range(2): + self.client.message(self.alice, self.chat, f"Сообщение {i}") + first = self.client.call("GET", self.path + "/messages?limit=1", actor=self.alice, schema="MessagePage")["json"] + cursor = first["next_cursor"] + other = self.client.call("POST", "/api/v1/chats", 201, actor=self.alice, schema="Chat", + body={"title": "Другой чат", "member_ids": [self.alice["id"]]})["json"] + path = "/api/v1/chats/" + other["id"] + "/messages?" + urlencode({"cursor": cursor}) + self.assert_error("GET", path, 400, "invalid_request", actor=self.alice) + data = json.loads(base64.urlsafe_b64decode(cursor)) + data["id"] = str(uuid4()) + changed = base64.urlsafe_b64encode(json.dumps(data).encode()).decode() + self.assert_error("GET", self.path + "/messages?" + urlencode({"cursor": changed}), + 400, "invalid_request", actor=self.alice) + + def test_framework_errors_and_request_ids(self): + first = self.assert_error("GET", "/unknown", 404, "not_found") + second = self.assert_error("GET", "/unknown", 404, "not_found") + self.assertNotEqual(first["request_id"], second["request_id"]) + self.assert_error("POST", "/health/live", 405, "invalid_request") diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..fe8c04e --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,135 @@ +import hashlib +import http.client +import io +import json +import struct +from urllib.parse import urlencode, urlsplit +from uuid import uuid4 +import zlib + +from PIL import Image + +from helpers import ApiTestCase, ROOT + + +class AttachmentTests(ApiTestCase): + def upload(self, data=None, filename="photo.png", content_type="image/png", expected=202, actor=None): + if data is None: + data = (ROOT / "tests/fixtures/sample.png").read_bytes() + return self.client.call("POST", self.path + "/attachments?" + urlencode({"filename": filename}), + expected, actor=actor or self.alice, raw=data, + headers={"Content-Type": content_type}, schema="Attachment" if expected == 202 else None) + + def test_png_and_jpeg_originals(self): + jpeg = io.BytesIO() + Image.new("RGB", (100, 60), "orange").save(jpeg, format="JPEG") + for data, mime, filename in (((ROOT / "tests/fixtures/sample.png").read_bytes(), "image/png", "Фото 👋.png"), + (jpeg.getvalue(), "image/jpeg", "photo.jpeg")): + with self.subTest(mime=mime): + attachment = self.upload(data, filename, mime)["json"] + self.assertEqual(attachment["filename"], filename) + self.assertEqual(attachment["status"], "queued") + attachment = self.client.wait_ready(self.bob, attachment) + self.assertEqual({v["name"] for v in attachment["variants"]}, {"thumbnail", "crop", "optimized"}) + self.assertIsNone(attachment["error_code"]) + self.assertEqual(attachment["sha256"], hashlib.sha256(data).hexdigest()) + self.assertEqual(attachment["size_bytes"], len(data)) + self.assertEqual(self.client.call("GET", "/api/v1/attachments/" + attachment["id"], + actor=self.bob, schema="Attachment")["json"], attachment) + result = self.client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=self.bob) + self.assertEqual(result["bytes"], data) + self.assertEqual(result["headers"].get_content_type(), mime) + self.assertEqual(result["headers"]["X-Content-Type-Options"], "nosniff") + + def test_corruption_and_mime_spoof(self): + png = (ROOT / "tests/fixtures/sample.png").read_bytes() + for data, mime in ((png, "image/jpeg"), (png[:50], "image/png"), + (b"not an image", "image/png"), (b"", "image/png"), + (b"GIF89a", "image/png"), (png, "application/octet-stream")): + with self.subTest(size=len(data), mime=mime): + result = self.upload(data, content_type=mime, expected=415) + self.assertEqual(result["json"]["error"]["code"], "unsupported_media_type") + + def test_filename_rejected_before_storage(self): + for filename in ("", " ", "../a.png", "a/b.png", "a\\b.png", "a\x00.png", "a\r\n.png", "a\x7f.png", "a" * 256): + with self.subTest(filename=repr(filename)): + result = self.upload(filename=filename, expected=400) + self.assertEqual(result["json"]["error"]["code"], "invalid_request") + self.client.call("POST", self.path + "/attachments", 400, actor=self.alice, raw=b"x", + headers={"Content-Type": "image/png"}) + + def test_content_length_limit_before_body(self): + self.client.call("POST", self.path + "/attachments?filename=a.png", 413, actor=self.alice, + raw=b"x", headers={"Content-Type": "image/png", "Content-Length": str(10485761)}) + + def test_chunked_limit_without_content_length(self): + target = urlsplit(self.client.base) + connection = http.client.HTTPConnection(target.hostname, target.port, timeout=15) + try: + connection.request("POST", self.path + "/attachments?filename=large.png", + body=iter([b"x" * 65536] * 160 + [b"x"]), + headers={"Authorization": "Bearer " + self.alice["token"], "Content-Type": "image/png"}, + encode_chunked=True) + response = connection.getresponse() + self.assertEqual(response.status, 413) + self.assertEqual(json.loads(response.read())["error"]["code"], "payload_too_large") + finally: + connection.close() + + def test_pixel_limit_before_decode(self): + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + # Размеры в корректном IHDR превышают лимит; pixel data вообще не выделяется. + data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 10000, 4001, 8, 2, 0, 0, 0)) + data += chunk(b"IDAT", b"") + chunk(b"IEND", b"") + self.upload(data, expected=413) + + def test_access_and_authentication(self): + attachment = self.upload()["json"] + for suffix in ("", "/content"): + path = "/api/v1/attachments/" + attachment["id"] + suffix + self.assert_error("GET", path, 401, "unauthorized") + self.assert_error("GET", path, 404, "not_found", actor=self.eve) + self.assert_error("GET", "/api/v1/attachments/" + str(uuid4()) + suffix, + 404, "not_found", actor=self.eve) + self.upload(expected=404, actor=self.eve) + self.client.call("POST", self.path + "/attachments?filename=../bad", 401, + raw=b"x", headers={"Content-Type": "image/png"}) + self.client.wait_ready(self.alice, attachment) + self.assert_error("GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=missing", + 404, "not_found", actor=self.alice) + + def test_attachment_only_message_and_other_member(self): + attachment = self.upload()["json"] + message = self.client.message(self.bob, self.chat, "", attachments=[attachment["id"]])["json"] + self.assertEqual(message["attachment_ids"], [attachment["id"]]) + self.assertEqual(message["sender_id"], self.bob["id"]) + self.assertEqual(self.read_all(self.path + "/messages", self.alice, "MessagePage"), [message]) + + def test_foreign_and_unknown_attachment_ids(self): + attachment = self.upload()["json"] + other = self.client.call("POST", "/api/v1/chats", 201, actor=self.alice, schema="Chat", + body={"title": "Другой чат", "member_ids": [self.alice["id"]]})["json"] + for chat_id, ids in ((self.chat["id"], [str(uuid4())]), (other["id"], [attachment["id"]])): + self.assert_error("POST", "/api/v1/chats/" + chat_id + "/messages", 404, "not_found", actor=self.alice, + body={"text": "", "attachment_ids": ids}) + + def test_invalid_attachment_lists(self): + attachment = self.upload()["json"] + for ids in ([attachment["id"], attachment["id"].upper()], [str(uuid4()) for _ in range(11)], + ["not-a-uuid"], [True], None, "uuid"): + self.assert_error("POST", self.path + "/messages", 400, "invalid_request", actor=self.alice, + body={"text": "Текст", "attachment_ids": ids}) + + def test_idempotency_includes_ordered_attachment_ids(self): + first, second = self.upload()["json"], self.upload()["json"] + key = uuid4().hex + ids = [first["id"], second["id"]] + saved = self.client.message(self.alice, self.chat, "", key=key, attachments=ids)["json"] + self.assertEqual(self.client.message(self.alice, self.chat, "", key=key, attachments=ids)["json"], saved) + for changed in ([second["id"], first["id"]], [first["id"]]): + self.assert_error("POST", self.path + "/messages", 409, "conflict", actor=self.alice, + body={"text": "", "attachment_ids": changed}, headers={"Idempotency-Key": key}) + key = uuid4().hex + saved = self.client.message(self.alice, self.chat, "Без файла", key=key)["json"] + self.assertEqual(self.client.message(self.alice, self.chat, "Без файла", key=key, attachments=[])["json"], saved) diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 0000000..abebb61 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,47 @@ +from concurrent.futures import ThreadPoolExecutor +import threading +from uuid import uuid4 + +from helpers import ApiTestCase, Client + + +class ConcurrencyTests(ApiTestCase): + def test_ten_clients_send_one_hundred_messages(self): + barrier = threading.Barrier(10) + + def send_messages(worker): + client = Client() + actor = self.alice if worker % 2 == 0 else self.bob + barrier.wait(timeout=10) + sent = [] + for index in range(10): + message = client.message(actor, self.chat, f"parallel-{worker}-{index}")["json"] + self.assertEqual(message["sender_id"], actor["id"]) + sent.append(message) + return sent + + with ThreadPoolExecutor(max_workers=10) as pool: + batches = list(pool.map(send_messages, range(10))) + sent = [message for batch in batches for message in batch] + history = self.read_all(self.path + "/messages", self.alice, "MessagePage", limit=17) + self.assertEqual(len(history), 100) + self.assertEqual(len({message["id"] for message in history}), 100) + self.assertEqual(history, sorted(sent, key=lambda item: (item["created_at"], item["id"]))) + self.assertEqual({message["text"] for message in history}, + {f"parallel-{worker}-{index}" for worker in range(10) for index in range(10)}) + print("PASS: 10 clients sent 100 messages; history contains exactly 100 unique messages.") + + def test_simultaneous_registration_of_one_username(self): + username = "race_" + uuid4().hex[:20] + barrier = threading.Barrier(10) + + def register(worker): + client = Client() + barrier.wait(timeout=10) + return client.call("POST", "/api/v1/users", [201, 409], + body={"username": username, "display_name": f"Клиент {worker}", "password": "Synthetic-password-123"})["status"] + + with ThreadPoolExecutor(max_workers=10) as pool: + statuses = list(pool.map(register, range(10))) + self.assertEqual(statuses.count(201), 1) + self.assertEqual(statuses.count(409), 9) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 0000000..a53fd30 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,62 @@ +from concurrent.futures import ThreadPoolExecutor +import json +import threading +from uuid import uuid4 + +from helpers import ApiTestCase, Client + + +class IdempotencyTests(ApiTestCase): + def test_parallel_replay_across_replicas(self): + key = uuid4().hex + barrier = threading.Barrier(20) + + def send(_): + client = Client() + barrier.wait(timeout=10) + return client.message(self.alice, self.chat, "Одно сообщение 👋", key=key) + + with ThreadPoolExecutor(max_workers=20) as pool: + responses = list(pool.map(send, range(20))) + self.assertGreaterEqual(len({r["headers"]["X-Instance-Id"] for r in responses}), 2) + self.assertTrue(all(r["json"] == responses[0]["json"] for r in responses)) + self.assertEqual(self.read_all(self.path + "/messages", self.bob, "MessagePage"), [responses[0]["json"]]) + + def test_key_format_and_missing_key(self): + for key in (None, "", " ", "a b", "a/b", "a" * 129): + with self.subTest(key=key): + headers = {} if key is None else {"Idempotency-Key": key} + self.client.call("POST", self.path + "/messages", 400, actor=self.alice, + body={"text": "Текст"}, headers=headers) + for key in ("A", "a" * 128, "ABCdef012._:-"): + self.client.message(self.alice, self.chat, key, key=key) + + def test_conflict_and_semantic_json_replay(self): + key = uuid4().hex + first = self.client.message(self.alice, self.chat, "Текст 👋", key=key)["json"] + replay = self.client.call("POST", self.path + "/messages", 201, actor=self.alice, + raw=json.dumps({"text": "Текст 👋"}, ensure_ascii=True, indent=2).encode(), + headers={"Content-Type": "application/json", "Idempotency-Key": key}, + schema="Message")["json"] + self.assertEqual(first, replay) + self.assert_error("POST", self.path + "/messages", 409, "conflict", actor=self.alice, + body={"text": "Другой"}, headers={"Idempotency-Key": key}) + self.assertEqual(self.read_all(self.path + "/messages", self.bob, "MessagePage"), [first]) + + def test_scope_is_user_and_chat(self): + key = uuid4().hex + first = self.client.message(self.alice, self.chat, "Алиса", key=key)["json"] + second = self.client.message(self.bob, self.chat, "Боб", key=key)["json"] + other = self.client.call("POST", "/api/v1/chats", 201, actor=self.alice, schema="Chat", + body={"title": "Другой чат", "member_ids": [self.alice["id"]]})["json"] + third = self.client.message(self.alice, other, "Другой чат", key=key)["json"] + self.assertEqual(len({first["id"], second["id"], third["id"]}), 3) + + def test_replay_rechecks_session_and_access(self): + key = uuid4().hex + self.client.message(self.alice, self.chat, "Текст", key=key) + self.client.call("POST", self.path + "/messages", 404, actor=self.eve, + body={"text": "Текст"}, headers={"Idempotency-Key": key}) + self.client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=self.alice) + self.client.call("POST", self.path + "/messages", 401, actor=self.alice, + body={"text": "Текст"}, headers={"Idempotency-Key": key}) diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py new file mode 100644 index 0000000..d95c4d4 --- /dev/null +++ b/student-tests/test_sessions.py @@ -0,0 +1,51 @@ +import base64 +from uuid import uuid4 + +from helpers import ApiTestCase + + +class SessionTests(ApiTestCase): + def basic(self, username, password): + value = base64.b64encode((username + ":" + password).encode()).decode() + return {"Authorization": "Basic " + value} + + def test_wrong_password_and_basic_only_for_login(self): + registration = self.alice["registration"] + self.assert_error("POST", "/api/v1/auth/sessions", 401, "unauthorized", + headers=self.basic(registration["username"], "wrong-password")) + self.assert_error("POST", "/api/v1/auth/sessions", 401, "unauthorized") + self.assert_error("POST", "/api/v1/auth/sessions", 401, "unauthorized", + headers={"Authorization": "Basic not-base64!"}) + self.assert_error("GET", "/api/v1/users/me", 401, "unauthorized", + headers=self.basic(registration["username"], registration["password"])) + + def test_logout_revokes_only_one_session(self): + registration = self.alice["registration"] + second = self.client.call("POST", "/api/v1/auth/sessions", 201, schema="Session", + headers=self.basic(registration["username"], registration["password"]))["json"] + self.client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=self.alice) + self.assert_error("GET", "/api/v1/users/me", 401, "unauthorized", actor=self.alice) + self.assert_error("DELETE", "/api/v1/auth/sessions/current", 401, "unauthorized", actor=self.alice) + self.client.call("GET", "/api/v1/users/me", schema="User", headers={"Authorization": "Bearer " + second["token"]}) + + def test_password_validation(self): + for password in ("short", "x" * 129, 123, None, " " * 12): + body = {"username": "pw_" + uuid4().hex[:20], "display_name": "Пароль", "password": password} + self.assert_error("POST", "/api/v1/users", 400, "invalid_request", body=body) + + def test_unicode_password_with_colon_and_private_response(self): + username = "pw_" + uuid4().hex[:20] + password = "Пароль:с:двоеточием-123" + user = self.client.call("POST", "/api/v1/users", 201, schema="User", + body={"username": username, "display_name": "Пароль", "password": password})["json"] + session = self.client.call("POST", "/api/v1/auth/sessions", 201, schema="Session", + headers=self.basic(username, password))["json"] + self.assertEqual(session["user"], user) + self.assertEqual(set(user), {"id", "username", "display_name", "created_at"}) + self.assertGreaterEqual(len(session["token"]), 32) + + def test_sql_like_message_is_plain_text(self): + text = "'; DROP TABLE messages; -- Привет" + message = self.client.message(self.alice, self.chat, text)["json"] + self.assertEqual(message["text"], text) + self.assertEqual(self.read_all(self.path + "/messages", self.bob, "MessagePage"), [message]) diff --git a/student-tests/verify_restart.py b/student-tests/verify_restart.py new file mode 100644 index 0000000..3f00da0 --- /dev/null +++ b/student-tests/verify_restart.py @@ -0,0 +1,37 @@ +"""Публичный persistence + доказательство реального restart каждого постоянного сервиса.""" +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def verify(env): + def run(*args): + return subprocess.check_output(args, cwd=ROOT, env=env, text=True, timeout=180).strip() + def snapshot(): + ids = run('docker', 'compose', 'ps', '-q').split() + infos = json.loads(run('docker', 'inspect', *ids)) + return {info['Id']: {'service': info['Config']['Labels']['com.docker.compose.service'], + 'started_at': info['State']['StartedAt'], + 'volumes': sorted(m['Name'] for m in info['Mounts'] if m['Type'] == 'volume')} + for info in infos} + before = snapshot() + print(run('make', 'test-persistence', 'PYTHON=' + sys.executable, 'ACTION_SCRIPT=scripts/restart.sh'), flush=True) + after = snapshot() + assert before.keys() == after.keys() + assert all(before[key]['started_at'] != after[key]['started_at'] for key in before) + assert all(before[key]['volumes'] == after[key]['volumes'] for key in before) + required = {'api', 'lb', 'postgres', 'valkey', 'broker', 's3', 's3-storage', 'cleanup', 'publisher', 'worker'} + required.update({'otel-collector', 'prometheus', 'loki', 'tempo', 'grafana'}) + assert required == {value['service'] for value in after.values()} + return {'passed': True, 'before': before, 'after': after, 'same_containers_and_volumes': True} + + +if __name__ == '__main__': + result = {'utc': datetime.now(timezone.utc).isoformat(), 'command': '.venv/bin/python student-tests/verify_restart.py', **verify(dict(os.environ))} + (ROOT / 'evidence/lab6-restart.json').write_text(json.dumps(result, indent=2) + '\n') + print('PASS: all persistent services actually restarted; container IDs and volumes preserved') -- 2.54.0