diff --git a/.dockerignore b/.dockerignore index bc16e55..5b43314 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,6 @@ __pycache__ node_modules evidence .DS_Store + +.lab6-faults +bin diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59c9f4e..9098144 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,20 @@ -## Лабораторная 6 +## Лабораторная 6 — БПИ2303 — Поликарпов — на 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор: **Поликарпов Даниил Александрович**, группа **БПИ2303**. +Трек 5: **SLO и обнаружение инцидентов**. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Из собственной lab5 перенесены Go API, миграции, transactional outbox, RabbitMQ worker, S3 pipeline и существующие проверки. Добавлены OTLP metrics/logs/traces, durable W3C context для retry/recovery, Collector, Prometheus/Loki/Tempo, Grafana provisioning/dashboard и burn-rate alerts. Controller выполняет parsing/validation, service — бизнес-операции/SLO, storage — сохранение; интеграция OTel находится в pkg/telemetry. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Запуск: `make up`, затем `make test test-go prove-correlation`. Полные prerequisites, конфигурация и воспроизводимые команды — в [REPORT.md](../REPORT.md). + +Результаты: 13/13 pipeline-сценариев PASS, persistence всех stores с сохранёнными volumes, API failover и privacy/trace checks PASS. В research: 74 094 запроса, 3529 вложений ready, 6334 подтверждённых сообщения сохранены; 0 firing на 6 коротких воздействиях, все ожидаемые alerts firing/resolved на 6 длительных. При 300s отказа Tempo бизнес-запросы продолжались; buffer/RAM и потери telemetry измерены. + +Ограничения: один host и единичные stores, учебный часовой SLO, best-effort telemetry; production HA/TLS/mTLS не заявляются. Источники, роль ИИ и два незасчитанных промежуточных прогона описаны в отчёте. Новые unit-тесты не добавлялись; запускались существующие проверки и новые сценарии измерений. + +- [x] Обязательные возможности сохранены; контракт и публичные tests не ослаблены. +- [x] make check/validate, build/vet и smoke выполнены; реальные результаты в evidence/. +- [x] Отказы выбранного уровня воспроизведены; есть raw observations, summary и runbook. +- [x] Секреты/приватные ключи вне Git; в доказательствах нет session tokens. +- [x] README задания сохранён; REPORT.md содержит инструкцию решения. + +Это заполненный шаблон описания; публикация PR в рамках этой работы не выполнялась. diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 0000000..d95549f --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,28 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.27.2' + cache-dependency-path: service/go.sum + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: pip install -r requirements-dev.txt + - run: make check validate build vet + - run: make up + - run: make test test-go prove-correlation observe + - if: always() + uses: actions/upload-artifact@v4 + with: + name: sanitized-evidence + path: evidence/ + - if: always() + run: make down diff --git a/.gitignore b/.gitignore index b5a7189..29789e6 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +.lab6-faults/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a953e59 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +# syntax=docker/dockerfile:1 + +FROM --platform=$BUILDPLATFORM golang:1.27.2-alpine3.23@sha256:9e45f0eb4a63ed37ad6e604950407558b7c738e34e015ae77a5d4ad369cde079 AS build + +WORKDIR /src + +COPY service/go.mod service/go.sum ./ +RUN go mod download + +COPY service/ ./ + +ARG TARGETOS +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -mod=readonly -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/app + +FROM alpine:3.23.6@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 + +RUN apk add --no-cache imagemagick=7.1.2.35-r0 imagemagick-webp=7.1.2.35-r0 imagemagick-jpeg=7.1.2.35-r0 +COPY infra/imagemagick-policy.xml /etc/ImageMagick-7/policy.xml + +COPY --from=build --chmod=0555 /out/messenger /usr/local/bin/messenger + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_SHUTDOWN_TIMEOUT=8s + +USER 65532:65532 +EXPOSE 8080 +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD wget -q -O /dev/null "http://127.0.0.1:${APP_PORT}/health/ready" || exit 1 + +ENTRYPOINT ["/usr/local/bin/messenger"] diff --git a/Makefile b/Makefile index 5d86d17..2b7e62b 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,25 @@ +SHELL := /bin/bash +GO ?= go PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://localhost:18086 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) +OPENAPI_SPEC := contracts/openapi.json +GEN_OUTPUT := service/internal/gen/contracts/types.gen.go +GEN_PACKAGE := contracts +OAPI_CODEGEN ?= $(shell $(GO) env GOPATH)/bin/oapi-codegen -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover openapi-generate certs key-gen create-dir gen-private-key gen-public-key gen-key gen-crt help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "observe/prove-correlation/experiment-slo: telemetry evidence and SLO research" + @echo "up/down: lab6 async image pipeline; build/vet: Go checks" + @echo "test: lab6 API smoke; test-go/test-storage: Go checks" + @echo "restart/restore/rolling-update: infrastructure actions; benchmark/load: synthetic traffic" + @echo "check/validate: lab6 contracts; test-persistence/test-failover: explicit ACTION_SCRIPT hooks" + @echo "reconcile/reconcile-apply: storage consistency" + @echo "openapi-generate: generate types from lab6 contract; key-gen/certs: local keys" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +40,99 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + + +openapi-generate: + @mkdir -p $(dir $(GEN_OUTPUT)) + @echo "Generating Go types from $(OPENAPI_SPEC)..." + @set -o pipefail; \ + $(PYTHON) scripts/generate-types.py | \ + $(OAPI_CODEGEN) -generate types -package $(GEN_PACKAGE) -o $(GEN_OUTPUT) /dev/stdin + +key-gen: gen-public-key + +create-dir: + @mkdir -p certs +gen-private-key: | create-dir + @openssl genrsa -out certs/private_key.pem 2048 +gen-public-key: gen-private-key + @openssl rsa -in certs/private_key.pem -pubout -outform PEM -out certs/public_key.pem + +certs: gen-crt + +gen-key: | create-dir + @openssl genrsa -out ./certs/server.key 2048 +gen-crt: gen-key + @openssl req -new -x509 -key ./certs/server.key -out ./certs/server.crt -days 365 + + +.PHONY: up down test-go +up: + ./scripts/bootstrap.sh + ./scripts/compose.sh up -d --build --wait --wait-timeout 120 + $(PYTHON) scripts/wait-ready.py + +down: + ./scripts/compose.sh down + +test-go: + cd service && TEST_BASE_URL="$(BASE_URL)" $(GO) test -race -count=1 -v ./... + +RATE ?= 100 +DURATION ?= 30s +BENCH_ARGS ?= + +.PHONY: build vet restart restore rolling-update benchmark test-storage +build: + cd service && $(GO) build ./... + @mkdir -p bin + cd service && $(GO) build -o ../bin/benchmark ./cmd/benchmark + +vet: + cd service && $(GO) vet ./... + +restart: + ./scripts/restart.sh + +restore: + ./scripts/compose.sh up -d --wait --wait-timeout 90 --no-recreate api lb + +rolling-update: + ./scripts/rolling-update.sh + +benchmark: build + ./bin/benchmark -base-url "$(BASE_URL)" -rate "$(RATE)" -duration "$(DURATION)" $(BENCH_ARGS) + +test-storage: + ./scripts/test-storage.sh + +.PHONY: reconcile reconcile-apply + +reconcile: + ./scripts/compose.sh run --rm -T operator storage reconcile + +reconcile-apply: + ./scripts/compose.sh run --rm -T operator storage reconcile --apply + +.PHONY: test-pipeline +test-pipeline: + $(PYTHON) student-tests/pipeline.py --base-url "$(BASE_URL)" --repeats 3 --output evidence/pipeline.json + +.PHONY: test-images test-staging +test-images: + $(PYTHON) student-tests/images.py + +test-staging: + $(PYTHON) student-tests/storage.py + +.PHONY: test-broker-persistence +test-broker-persistence: + $(PYTHON) student-tests/broker_persistence.py + +.PHONY: observe prove-correlation experiment-slo +observe: + $(PYTHON) scripts/observations.py +prove-correlation: + $(PYTHON) scripts/prove-correlation.py +experiment-slo: + $(PYTHON) scripts/experiment-slo.py diff --git a/REPORT.md b/REPORT.md index b97ae42..8a1c4fd 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,219 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор: **Поликарпов Даниил Александрович**. +- Группа: **БПИ2303**. +- Целевая оценка: **5**, трек 2 — SLO и обнаружение инцидентов. +- Ветка: `solution/bpi2303-polikarpov`. В репозитории есть commit `8b381c1` («Реализация на 5»); завершившиеся после него измерения и итоговый отчёт дополнены в рабочем дереве. Новый commit и PR на этапе завершения не создавались. Базовая ревизия, fingerprint исходников и image IDs фиксируются в `evidence/environment.json` и результатах исследования. +- Основа: собственная реализация соседней lab5. Использованы Go, pgx, AWS SDK, RabbitMQ amqp091-go, ImageMagick, OpenTelemetry Go и Grafana stack. Перенос, подготовка отчета и проверка выполнены с помощью Codex. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker с Compose, Python 3.9+ и для локальной сборки Go 1.27+. Контейнерная сборка закреплена на Go 1.27.2 / Alpine 3.23.6; ImageMagick 7.1.2.35-r0. Образы и библиотеки имеют фиксированные версии; фактические image IDs и ресурсы приведены в `evidence/environment.json`. -## Архитектура и решения +```sh +make check build vet +make up +make test test-go +make prove-correlation observe +python3 scripts/prove-signals.py +make test-storage +make test-images test-staging test-broker-persistence +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +make restore +python3 student-tests/pipeline.py --repeats 1 --output evidence/pipeline.json +make experiment-slo +python3 scripts/summarize-slo.py +python3 scripts/analyze-backend.py +python3 scripts/prove-persistence.py +python3 scripts/capture-environment.py +make down +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Все сценарии отказов запускаются последовательно, отдельно от исследования. Полное исследование длительное: реальное ожидание окон, три повтора, без ускорения часов и подмены метрик. + +Полная валидация контракта: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make validate PYTHON=.venv/bin/python +``` + +`make up` идемпотентно создаёт локальные синтетические credentials в `.env.lab6` и `secrets/lab6`, запускает отдельный migrator, bootstrap приватного bucket и provisioning Grafana. Существующие credentials не ротируются. Новая конфигурация задаёт SESSION_TTL_SECONDS=14400 для длительного исследования; стандартный контрактный default без настройки остаётся 3600. Пароли и ключи находятся вне Git и не входят в Docker build context. + +API: http://127.0.0.1:18086. Grafana: http://127.0.0.1:13006/d/messenger-lab6, login `admin`; пароль читается локально из `secrets/lab6/grafana_password` и не включается в доказательства. + +Обычная остановка `make down` сохраняет данные. Намеренное удаление **только учебных данных lab6**: `./scripts/compose.sh down --volumes`; автоматически эта команда не выполняется. Стенд lab5 на 18085 имеет отдельные проект, volumes и секреты. Первый локальный запуск перенесённого Go suite ещё использовал старый default URL и создал синтетические записи в lab5; адрес исправлен, результат не засчитывался, проверки повторены на lab6. + +## Архитектура и границы + +```mermaid +flowchart LR + Client --> LB[HAProxy] + LB --> API[API x2] + API --> S3[Private S3 staging/results] + API --> DB[PostgreSQL metadata/jobs/outbox/SLO] + DB --> Dispatcher + Dispatcher --> Rabbit[RabbitMQ] + Rabbit --> Worker + Worker --> S3 + Worker --> DB + API --> Collector[OTel Collector] + Worker --> Collector + Dispatcher --> Collector + Collector --> Prometheus + Collector --> Loki + Collector --> Tempo + Prometheus --> Grafana + Loki --> Grafana + Tempo --> Grafana +``` + +Handler разбирает HTTP, проверяет поля и ограничивает входной поток. Service проверяет доступ, управляет бизнес-состояниями, транзакциями, retries и решением good/bad для SLO. Repo/storage выполняет чтение и сохранение фактов; SQL-агрегаты формируют снимок состояния. `pkg/postgres` содержит только настройку доступа/пула, timeout и подключение безопасного tracer. Новая интеграция SDK расположена в `internal/pkg/telemetry`; обработка изображений и S3 также остаются в своих pkg. + +Из lab5 перенесены API, сессии, cache, идемпотентность сообщений, durable staging, transactional outbox, RabbitMQ, worker, lease/retry/recovery, versioned manifest и операторские команды. Бизнес-контракт lab6 совпадает с lab5; сохранены контракт и публичные проверки текущего skeleton. Старые README/REPORT, секреты и evidence не переносились как результаты lab6. + +До 202 проверенный оригинал находится в приватном S3 staging, а metadata/job/outbox фиксируются одной транзакцией. Worker публикует оригинал и варианты в S3 и переводит job в ready одной DB-транзакцией. At-least-once доставка и идемпотентный эффект сохранены; exactly-once доставка не заявляется. S3 staging и results — префиксы одного сервиса, а не независимые отказоустойчивые хранилища. + +Миграция 010 добавляет durable trace carrier и таблицу однократных SLO-наблюдений; 011 — факт terminal failure. Метаданные готовности и terminal error записываются вместе с соответствующим бизнес-переходом. Observer в dispatcher оценивает deadline даже при остановленном worker. Повторная доставка/replay не отменяет уже зафиксированное нарушение. + +### Компоненты и эксплуатация + +| Роль → service | Версия | Readiness / persistent state / доступ | +| --- | --- | --- | +| API → api, две реплики | Go 1.27.2, собственный image | `/health/ready`: DB + S3; без durable local state, порты реплик закрыты | +| LB → lb | HAProxy 3.2-alpine, digest в Compose | HTTP readiness API; localhost:18086 | +| DB → postgres | PostgreSQL 17.11, digest | pg_isready; postgres_data; локальный пароль env | +| Cache → redis | Redis 8.2.11, digest | authenticated PING; redis_data/AOF; необязателен для корректности | +| S3 → s3 | SeaweedFS 4.52, digest | bootstrap проверяет запись; s3_data/internal network; credentials files | +| Broker → broker | RabbitMQ 4.2.0, digest | check_running; broker_data; durable queue/persistent delivery/confirms/ack | +| Worker / dispatcher | собственный image | отдельные процессы/reconnect; прогресс по jobs/observations, без HTTP healthcheck | +| Collector → otel-collector | image tag 0.140.0, binary сообщает 0.140.1 | :13133; ограниченные недолговечные очереди, не хранилище истории | +| Metrics → prometheus | 3.16.0 | /-/ready; prometheus_data; retention 7d / 1 GiB | +| Logs → loki | 3.7.10 | /ready; loki_data, TSDB v13/filesystem; retention 7d | +| Traces → tempo | 2.9.0 | /ready; tempo_data, WAL/local blocks; retention 7d | +| UI → grafana | 13.3.1 | /api/health; grafana_data + provisioning; localhost:13006, пароль file | + +Внутренние OTLP/metrics/storage/admin порты не публикуются. `scripts/wait-ready.py` проверяет backend endpoints через внутреннюю сеть, а не только состояние running. `scripts/restart.sh` перезапускает API/worker/dispatcher и все бизнес- и telemetry-хранилища с volumes; жизненные события сохранены отдельно. Grafana — интерфейс, Collector — обработчик/экспортёр; долговременные сигналы хранятся в Prometheus/Loki/Tempo. + +`.env.example` содержит логические рекомендации skeleton. Реальная конфигурация использует APP_*, POSTGRES_*, REDIS_*, SESSION_TTL_SECONDS, S3_* и *_FILE, BROKER_URL_FILE, WORKER_*, JOB_*, STAGING_RETENTION и OTEL_EXPORTER_OTLP_ENDPOINT/PROTOCOL. DATABASE_URL_FILE/CACHE_URL_FILE/SESSION_STORE из примера не используются: PostgreSQL настраивается отдельными полями, сессии хранятся в DB, Redis — кэш. Секундные JOB_TIMEOUT_SECONDS/STAGING_RETENTION_SECONDS сопоставлены duration-настройкам JOB_TIMEOUT/STAGING_RETENTION. Имена OTel services задаются ролью процесса, resource attributes — кодом Init. + +### Сигналы и связь процессов + +SDK: OTel Go 1.47.0, OTLP logs exporter 0.23.0, slog bridge 0.21.0; собственная схема instrumentation `messenger`; выбранное подмножество стандартных имён атрибутов закреплено по Semantic Conventions **1.37.0** (входит в закреплённый OTel Go SDK). Использованы имена атрибутов OTel для ресурсов/HTTP/DB; публичные метрики приложения имеют документированную собственную схему и не дублируют стандартные инструменты. + +| Сигнал | Реальное имя в Prometheus / поля | Единицы и labels | +| --- | --- | --- | +| HTTP count / duration | messenger_http_requests_total / messenger_http_duration_seconds_bucket | counter: method/route/status_class; duration: method/route, секунды | +| Jobs | messenger_jobs_total | job_type, outcome=ready/retry/failed/duplicate | +| Processing duration | messenger_job_duration_seconds_bucket | job_type, outcome; секунды | +| Time to ready | messenger_job_ready_duration_seconds_bucket | job_type; секунды, после COMMIT | +| Queue | messenger_queue_depth / messenger_queue_oldest_age_seconds | queue; Rabbit ready count / возраст логической незавершённой job | +| Logical backlog | messenger_jobs_pending / messenger_outbox_pending | число jobs / публикаций, не взаимозаменяемы с broker depth | +| DB pool | messenger_db_pool_used/capacity/waiting; messenger_db_acquire_duration_seconds_bucket; messenger_db_locks_waiting | соединения, ожидающие acquire / секунды | +| Worker saturation | messenger_worker_busy/capacity | активные attempts / доступные slots | +| Process resources | messenger_process_memory / messenger_process_cpu | RSS bytes / накопленные CPU seconds | +| Ready SLO / freshness | messenger_ready_slo_total / messenger_observation_timestamp | outcome good/bad / Unix seconds | +| Export health | otelcol_* и messenger_telemetry_export_errors | accepted/sent/failed, queue size/capacity / SDK errors | +| Logs | timestamp, severity, service.name, event, trace_id, span_id, request_id | технические IDs только вне labels метрик | + +Histograms: buckets 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30, 60, 120, 300 seconds. Пример p95: `histogram_quantile(0.95, sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))`. Resource attributes: service.name/version/instance.id, deployment.environment.name=lab. Default sampling — ParentBased(AlwaysSample). + +W3C context сохраняется вместе с job/outbox. Dispatcher создаёт producer span из durable context и помещает его carrier в сообщение, worker создаёт дочерний consumer span. Retry наследует consumer context; recovery извлекает его из job. SQL-параметры, raw URL, Authorization, body и object keys не записываются. Error values редактируются до безопасной категории. В Loki service.instance.id и version хранятся как structured metadata; индексируются service.name и environment. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Критерий | Команда / наблюдение | Доказательство | +| --- | --- | --- | +| Целостность и схемы | make check validate | evidence/contracts.txt | +| Сборка и статические проверки | make build vet | evidence/build.txt | +| Публичный API и прежние Go-проверки с telemetry | make test test-go | evidence/final-checks.txt | +| Изображения, staging, broker persistence | make test-images test-staging test-broker-persistence | evidence/previous-lab-checks.txt и соответствующие JSON | +| Storage transactions | make test-storage | evidence/storage-checks.txt | +| Persistence с прежним токеном | make test-persistence | evidence/persistence.txt, evidence/persistence-processes.json | +| API failover | make test-failover; затем make restore | evidence/failover.txt, evidence/failover-action.jsonl | +| API → dispatcher → worker, retry, S3/DB, logs | make prove-correlation | evidence/correlation.json, correlation-check.txt | +| Реальные метрики, безопасные labels, canary, dashboard queries | python3 scripts/prove-signals.py | evidence/signals.json, signal-checks.txt | +| Config/provisioning | promtool, Collector validate, Loki verify; readiness | evidence/prometheus-config.txt, collector-config.txt, loki-config.txt; infra/otel/ | +| SLI/SLO и alerts | реальные recording/alert rules, общий dashboard | infra/otel/alerts.yaml, RUNBOOK.md | +| Два инцидента и исследование | make experiment-slo; summarize-slo.py | evidence/slo-experiment.json, slo-summary.json | +| Отказ telemetry backend | остановка Tempo под той же нагрузкой | evidence/backend-outage.json, backend-checks.txt | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Итоговый прогон завершён успешно. Сводная проверка полноты: `evidence/research-validation.json`; исходные наблюдения не заменены расчётными или искусственными сигналами. В `evidence/pipeline.json` подтверждены 13/13 сценариев. После них повторно прошли smoke, существующие Go-проверки с race detector, correlation и privacy/dashboard checks. Проверка failover восстановила запись за 1.148s; 133 записи подтверждены, один переходный сбой учтён. Persistence подтвердил перезапуск всех восьми хранилищ с прежними volumes и сохранение старой сессии. -## Эксперименты и отказы +## Методика исследования -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза: правила burn-rate с двумя окнами отсекают краткий всплеск и обнаруживают устойчивую деградацию; независимый deadline observer выявляет нарушение ready-SLO при полном отсутствии завершённых jobs. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +SLO: 99% бизнес-ответов без 5xx и 99% принятых jobs до ready ≤30s, окно 1h. Выбор 99% — учебный бюджет 1% событий за час: при постоянном профиле это соответствует примерно 36 секундам полного отказа. Он допускает единичный 5s всплеск, но не 240s деградацию. Для маленького fixture нормальная обработка существенно короче 30s; такой ready-бюджет оставляет запас на краткий backlog и несколько быстрых retries. Это целевой SLO для данного профиля, а не гарантия для любого изображения/нагрузки; публичный timeout smoke 60s остаётся отдельным условием. -## Ограничения и следующие шаги +Полные формулы, minimum traffic, timestamps и пределы интерпретации — в [runbook](infra/otel/RUNBOOK.md). Fast: 1m/5m, >6×, for20s. Slow: 2m/10m, >3×, for30s. Отдельно QueueLag >30s for20s и stale observations. Уведомления наружу не отправляются. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Нагрузка: open-loop 10 message requests/s, 90% GET history / 10% POST message, плюс fixture PNG 320×200 каждые 2s, in-flight ≤64, HTTP timeout6s, без retries. Перцентили успешных ответов включают scheduling delay; неуспешные статусы и пропуски генератора учитываются отдельно. RPS не является поиском максимальной пропускной способности. + +После обнаружения ограничения S3 первый прогон остановлен и сохранён как `evidence/aborted-capacity-*`; причины и исправление описаны в `evidence/aborted-capacity-diagnosis.md`. Второй промежуточный прогон выявил потерю первого всплеска в rate для новых counter series; причина и исправление: `evidence/aborted-counter-diagnosis.md`. Итоговые counters заранее публикуют нули, duration histogram не разделяется по status class; pool=16 на процесс. + +Для итогового прогона S3 допускает 16 volume slots с лимитом volume 1024 MiB, без удаления существующих данных. + +Прогрев 15 минут. Три повтора остановки worker и блокировки messages отдельной ограниченной DB-транзакцией: короткое воздействие 5s, длительное 240s, после длительного — 660s восстановления с трафиком, чтобы очистить 10m окно. Короткие воздействия разделены 60s. Затем Tempo останавливается на 300s при работающих metrics/logs pipelines. Состояния alerts и метрики снимаются каждые ~5s, Docker CPU/RAM — каждые ~30s. Цикл включает время сбора, поэтому точность обнаружения ограничена фактическим интервалом наблюдений, SDK export/scrape/evaluation и планированием процессов. + +Причины сначала устанавливаются по сохранённым сигналам, затем сопоставляются с отдельными timestamps инъекции. Для worker ожидается рост backlog/age без ухудшения HTTP; для DB — 5xx и длительные DB spans при работающем worker; для Tempo — экспортные ошибки/потери при сохранении бизнес-функций. + +## Фактические результаты исследования + +Прогон выполнен **16 ноября 2026, 08:17:58–10:16:11 UTC**, длительность **7092.82s (1ч 58мин 13с)**. Итоговая сверка артефактов выполнена 17 ноября. Аппаратная конфигурация: Apple M4 Pro, 14 CPU, host RAM 24 GiB; Docker VM — 14 CPU, 7.75 GiB. Лимиты приложены в `evidence/environment.json`. У генератора не было пропусков, у сборщика — ошибок чтения наблюдений. + +Всего **74 094** запроса: **66 860** успешных и **7 234** ответа 503 при намеренных воздействиях на messages. Таймаутов клиента, 401 и 429 в итоговом прогоне нет. Все **6 334** подтверждённых сообщения найдены в PostgreSQL; все **3 529** принятых вложений в конце достигли ready. Исходные PNG — по 111 817 bytes; принятый объём оригиналов **394 602 193 bytes**. История сообщений росла в ходе прогона; профиль читает первую страницу, а не доказывает максимальную пропускную способность на произвольном объёме данных. + +| Интервал | Успешные RPS | Доля ошибок | p50 / p95 / p99 успешных ответов, ms | +| --- | ---: | ---: | --- | +| Прогрев, 900s | 10.50 | 0% | 8.37 / 16.35 / 25.75 | +| Worker остановлен, повтор 1, 240s | 10.50 | 0% | 8.79 / 17.55 / 28.45 | +| DB lock, повтор 1, 240s | 0.50 | 95.2% | 27.06 / 41.26 / 48.94 | +| Tempo остановлен, 300s | 10.50 | 0% | 6.96 / 16.46 / 25.76 | + +Во время DB lock успешными оставались главным образом upload; низкий percentile **успешных** ответов не означает здоровую работу messages. Медленные ошибки видны в HTTP histogram и DB spans. Для всего прогона p50/p95/p99 успешных ответов: 7.84/17.53/29.31ms. Интервальные статусы классифицируются по моменту завершения ответа; обращения на границах фаз могут завершиться между операторскими действиями. Все ответы входят в общий итог. + +### Обнаружение и восстановление, три повтора + +Время ниже — первое сохранённое наблюдение состояния, а не точный момент пересечения порога. Точность ограничена экспортом, scrape/evaluation и опросом; дробные исходные значения находятся в `evidence/slo-summary.json`. + +| Alert | До первого firing, s: минимум–максимум | До resolved после восстановления, s: минимум–максимум | +| --- | ---: | ---: | +| QueueLag | 59.91–64.88 | 12.54–17.60 | +| ReadyBurnRateFast | 72.20–81.44 | 64.20–72.46 | +| ReadyBurnRateSlow | 87.29–91.54 | 123.94–129.35 | +| ApiBurnRateFast | 44.78–53.87 | 55.14–64.97 | +| ApiBurnRateSlow | 54.89–63.98 | 119.53–124.81 | + +На **6 коротких воздействиях** firing не наблюдался; на **6 длительных** все ожидаемые alerts сработали и затем разрешились. Это результат данного профиля и трёх повторов, а не доказательство нулевой вероятности ложных срабатываний. Отдельное сравнение с правилами без второго окна/без for не проводилось, поэтому вклад каждого механизма фильтрации из этого эксперимента не выделяется. Целевой SLO во время намеренных деградаций нарушается — обнаружение этого нарушения и проверяется. + +### Диагноз по сохранённым сигналам + +- **Worker:** на каждом длительном интервале растут возраст и backlog, новых `job_received` нет, upload продолжает получать 202, API burn rate равен нулю. В первой сохранённой выборке — 35 успешных upload и 0 событий потребления. После запуска появляются consumer/S3/DB spans, очередь убывает, jobs становятся ready. Ready counter учитывал bad по deadline ещё до завершения jobs. +- **DB:** одновременно наблюдаются около 20 ожиданий Lock, HTTP 5xx и DB client spans длительностью около двух секунд (первая сохранённая трасса — `postgresql SELECT` 1995.4ms). При этом ready burn rate остаётся нулевым. Это указывает на ожидание БД, а не на преобразование изображений или медленный код HTTP handler. Конкретная операторская блокировка messages сверена с журналом воздействия. + +Девять компактных наборов фактов (worker stop, recovery и DB wait для каждого повтора) сохранены в `evidence/incident-signals.json`; ошибок их сбора нет. [Runbook](infra/otel/RUNBOOK.md) задаёт те же шаги диагностики для другого студента. + +### Отказ Tempo и границы буферизации + +За 300s недоступности Tempo генератор получил 3150 успешных ответов: 2700 чтений, 300 сообщений и 150 upload; ошибок нет. После запуска Tempo вновь доступны traces. Очередь exporter достигла **32/32 запросов** и после восстановления стала **0**. Максимальная наблюдавшаяся память Collector — **93.84 MiB** при лимите контейнера **256 MiB**; в этот интервал его CPU составлял медианно 0.61%, максимум 3.51% одного ядра по выборкам Docker. API-реплики использовали примерно 70–77 MiB, dispatcher — 15 MiB, worker — 16 MiB в последних выборках этого интервала. + +По счётчикам за outage/recovery/drain: принято 51 598 spans, отправлено 13 416, разница — **около 38 182**. Зафиксированы 35 676 enqueue-failed spans и 2524 dropped items в отдельных записях exporter. Баланс приблизительный: остаются фоновые health/observer spans и несколько in-flight batches. Он не включает возможные потери до Collector; send-failed нельзя механически считать уникальными потерями. Telemetry в этом отказе терялась, бизнес-данные сохранились. Подробности: `evidence/backend-outage.json`. + +Ограничение очереди подтверждается одновременно конфигурацией и наблюдаемым плато; пятиминутная выборка RAM не является доказательством отсутствия любого возможного memory leak на произвольной длительности. CPU/RAM снимались примерно раз в 30s и могли пропустить короткие пики. + +### Версии артефактов + +Research записал OCI descriptor `d661ef49…`, повторный pipeline — `c7021c3f…`. Compose добавляет разные service labels и attestations при сборке одного Dockerfile для ролей, поэтому descriptor не является хешем бинарника. Платформенный manifest текущего dispatcher совпал с manifest в сохранённом логе сборки перед research; SHA-256 исполняемого файла — `33c6b59d…`. Доказательство: `evidence/image-provenance.json`; полный fingerprint исходников и фактические container images — в `evidence/environment.json`. + +## Ограничения + +Один host, PostgreSQL, S3, broker и LB остаются точками отказа. HA хоста/БД и потеря volumes не проверяются. CPU/RAM ограничены Compose; рядом работал старый стенд lab5, поэтому результаты относятся к этой конфигурации и не являются универсальным benchmark. + +HTTP SLI считает ответы приложения: сетевые отказы/LB 503 измеряются генератором отдельно, а 4xx входят в denominator. Ready-SLI относится к серверной публикации результата, не скачиванию файла клиентом. Пока DB недоступна, observer не может зафиксировать новые результаты, что должно быть видно по freshness. Клиентские timestamps и серверные timestamps имеют разные границы измерения. accepted_at и ready_at записываются внутри соответствующих транзакций; классификация непосредственно возле 30s зависит от длительности COMMIT. Histogram percentiles также являются интерполяцией внутри заданных buckets, а не точными клиентскими квантилями. + +Telemetry best effort: ограниченные SDK/Collector очереди допускают потерю сигналов при долгом отказе; sent-failed считает попытки, а не уникальные потерянные spans. Рестарт теряет недолговечные очереди. Retention ограничивает срок хранения; для Loki/Tempo нет отдельной гарантии дисковой квоты/backup. Credentials учебные, HTTP допускается только на localhost. TLS/mTLS и управление секретами относятся к lab7. + +Источники: [OpenTelemetry Go](https://opentelemetry.io/docs/languages/go/), [Collector configuration](https://opentelemetry.io/docs/collector/configuration/), [Loki native OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/), [Prometheus histograms](https://prometheus.io/docs/practices/histograms/), [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability). diff --git a/compose.faults.yaml b/compose.faults.yaml new file mode 100644 index 0000000..8bbd5b6 --- /dev/null +++ b/compose.faults.yaml @@ -0,0 +1,22 @@ +# Explicit isolated experiment profile. Never enabled by make up. +services: + api: + environment: + LAB5_FAULTS: "1" + volumes: + - ./.lab6-faults:/faults + worker: + environment: + LAB5_FAULTS: "1" + JOB_TIMEOUT: 5s + JOB_LEASE: 8s + volumes: + - ./.lab6-faults:/faults + dispatcher: + environment: + LAB5_FAULTS: "1" + volumes: + - ./.lab6-faults:/faults + operator: + environment: + LAB5_FAULTS: "1" diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..8dd2a4d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,297 @@ +name: mtusi2026lab6 +x-api: &api + image: ${API_IMAGE:-mtusi-messenger:lab6} + build: . + restart: unless-stopped + cpus: ${API_CPUS:-1} + mem_limit: ${API_MEMORY:-512m} + environment: &environment + APP_ENABLED: "true" + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_READ_TIMEOUT: 125s + APP_WRITE_TIMEOUT: 125s + APP_IDLE_TIMEOUT: 60s + APP_SHUTDOWN_TIMEOUT: 8s + APPLICATION_LOG_LEVEL: ${APPLICATION_LOG_LEVEL:-info} + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + APPLICATION_ENVIRONMENT: prod + APPLICATION_SHUTDOWN_TIMEOUT: 8s + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run scripts/bootstrap.sh} + POSTGRES_DATABASE: messenger + POSTGRES_VERSION_TABLE: versions + POSTGRES_RUN_MIGRATIONS: "false" + POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-16} + POSTGRES_CONNECT_TIMEOUT: 500ms + POSTGRES_QUERY_TIMEOUT: 2s + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 0 + REDIS_USERNAME: default + REDIS_PASSWORD: ${REDIS_PASSWORD:?Run scripts/bootstrap.sh} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + JWT_ACCESS_TOKEN_LIFETIME: 15m + JWT_PRIVATE_KEY_PATH: /run/certs/private_key.pem + JWT_PUBLIC_KEY_PATH: /run/certs/public_key.pem + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret + BROKER_URL_FILE: /run/secrets/broker_url + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT: ${JOB_TIMEOUT:-60s} + JOB_LEASE: ${JOB_LEASE:-90s} + STAGING_RETENTION: ${STAGING_RETENTION:-24h} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-120s} + RECONCILE_GRACE_PERIOD: ${RECONCILE_GRACE_PERIOD:-1h} + GOMEMLIMIT: 384MiB + networks: [default, storage] + volumes: + - ./secrets/lab6/broker_url:/run/secrets/broker_url:ro + - ./secrets/lab6/private_key.pem:/run/certs/private_key.pem:ro + - ./secrets/lab6/public_key.pem:/run/certs/public_key.pem:ro + - ./secrets/lab6/s3_api_access:/run/secrets/s3_access:ro + - ./secrets/lab6/s3_api_secret:/run/secrets/s3_secret:ro + stop_grace_period: 10s +services: + api: + <<: *api + deploy: + replicas: ${API_REPLICAS:-2} + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + migrate: + <<: *api + # The migration command uses only PostgreSQL: no signing keys or Redis credentials. + volumes: [] + environment: + <<: *environment + REDIS_PASSWORD: "" + restart: "no" + command: [migrate] + cpus: 0.5 + mem_limit: 128m + healthcheck: + disable: true + depends_on: + postgres: + condition: service_healthy + broker: + hostname: broker + image: rabbitmq:4.2.0-alpine@sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524 + restart: unless-stopped + cpus: 0.5 + mem_limit: 512m + environment: + RABBITMQ_DEFAULT_USER: messenger + RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?Run scripts/bootstrap.sh} + volumes: + - broker_data:/var/lib/rabbitmq + healthcheck: + test: [CMD, rabbitmq-diagnostics, -q, check_running] + interval: 3s + timeout: 5s + retries: 30 + stop_grace_period: 30s + worker: + <<: *api + command: [worker] + cpus: 1 + mem_limit: 1g + stop_grace_period: 10s + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + dispatcher: + <<: *api + command: [dispatcher] + cpus: 0.25 + mem_limit: 128m + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + lb: + image: ${HAPROXY_IMAGE:-haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0} + restart: unless-stopped + cpus: 0.5 + mem_limit: 128m + ports: + - "127.0.0.1:${HOST_PORT:-18086}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_started + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready] + interval: 2s + timeout: 1s + retries: 5 + stop_grace_period: 10s + postgres: + image: postgres:17.11-alpine3.23@sha256:05aa77262db9277fde6374bb2f901c9604b8a9373029e1a7993440c6aae48736 + restart: unless-stopped + cpus: 2 + mem_limit: 1g + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run scripts/bootstrap.sh} + POSTGRES_DB: messenger + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 1s + retries: 30 + stop_grace_period: 30s + redis: + image: redis:8.2.11-alpine3.22@sha256:489a769e0a54dc245896132a15c72d2bc7296719954da99398f43a3877e5f8eb + restart: unless-stopped + cpus: 0.25 + mem_limit: 128m + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD:?Run scripts/bootstrap.sh} + command: [sh, -c, 'exec docker-entrypoint.sh redis-server --appendonly yes --maxmemory 64mb --maxmemory-policy allkeys-lru --requirepass "$$REDIS_PASSWORD"'] + volumes: + - redis_data:/data + healthcheck: + test: [CMD-SHELL, 'REDISCLI_AUTH="$$REDIS_PASSWORD" redis-cli --no-auth-warning ping | grep -qx PONG'] + interval: 2s + timeout: 1s + retries: 20 + stop_grace_period: 10s + s3: + image: chrislusf/seaweedfs:4.52@sha256:57877e32bb90073b37dc109b438ba66e84fd5ed9ced5f5c04f56444ef243ce59 + restart: unless-stopped + cpus: 1 + mem_limit: 512m + networks: [storage] + command: [server, '-dir=/data', '-ip=s3', '-ip.bind=0.0.0.0', '-volume.max=16', '-master.volumeSizeLimitMB=1024', '-filer', '-s3', '-s3.port=9000', '-s3.config=/etc/seaweedfs/s3.json'] + volumes: + - s3_data:/data + - ./secrets/lab6/s3.json:/etc/seaweedfs/s3.json:ro + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:9333/cluster/status] + interval: 2s + timeout: 1s + retries: 30 + stop_grace_period: 30s + s3-init: + <<: *api + restart: "no" + cpus: 0.25 + mem_limit: 128m + command: [storage, bootstrap] + volumes: + - ./secrets/lab6/s3_admin_access:/run/secrets/s3_access:ro + - ./secrets/lab6/s3_admin_secret:/run/secrets/s3_secret:ro + healthcheck: + disable: true + depends_on: + s3: + condition: service_healthy + operator: + <<: *api + profiles: [tools] + restart: "no" + command: [storage, reconcile] + volumes: + - ./secrets/lab6/s3_admin_access:/run/secrets/s3_access:ro + - ./secrets/lab6/s3_admin_secret:/run/secrets/s3_secret:ro + - ./tests/fixtures:/fixtures:ro + - ./secrets/lab6/broker_url:/run/secrets/broker_url:ro + healthcheck: + disable: true + depends_on: {} + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.140.0 + restart: unless-stopped + command: [--config=/etc/otelcol/config.yaml] + cpus: 0.5 + mem_limit: 256m + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro + prometheus: + image: prom/prometheus:v3.16.0 + restart: unless-stopped + command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=7d, --storage.tsdb.retention.size=1GB, --web.enable-lifecycle] + cpus: 0.5 + mem_limit: 384m + volumes: + - ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro + - prometheus_data:/prometheus + loki: + image: grafana/loki:3.7.10 + restart: unless-stopped + command: [-config.file=/etc/loki/config.yaml] + cpus: 0.5 + mem_limit: 384m + volumes: + - ./infra/otel/loki.yaml:/etc/loki/config.yaml:ro + - loki_data:/loki + tempo: + image: grafana/tempo:2.9.0 + restart: unless-stopped + command: [-config.file=/etc/tempo.yaml] + user: '0:0' + cpus: 0.5 + mem_limit: 384m + volumes: + - ./infra/otel/tempo.yaml:/etc/tempo.yaml:ro + - tempo_data:/var/tempo + grafana: + image: grafana/grafana:13.3.1 + restart: unless-stopped + cpus: 0.5 + mem_limit: 384m + ports: ['127.0.0.1:13006:3000'] + environment: + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_password + GF_AUTH_ANONYMOUS_ENABLED: 'false' + GF_ANALYTICS_REPORTING_ENABLED: 'false' + GF_ANALYTICS_CHECK_FOR_UPDATES: 'false' + volumes: + - grafana_data:/var/lib/grafana + - ./secrets/lab6/grafana_password:/run/secrets/grafana_password:ro + - ./infra/otel/grafana:/etc/grafana/provisioning:ro + - ./infra/otel/dashboards:/var/lib/grafana/dashboards:ro + +volumes: + prometheus_data: + loki_data: + tempo_data: + grafana_data: + postgres_data: + redis_data: + s3_data: + broker_data: + +networks: + storage: + internal: true diff --git a/evidence/README.md b/evidence/README.md index 8df7001..9df79fc 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -3,3 +3,17 @@ Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + + +## Итоговый набор lab6 + +- `research-validation.json`, `slo-summary.json` — полнота и времена обнаружения/восстановления. +- `slo-experiment.json`, `slo-run.txt` — исходные наблюдения, нагрузка и ресурсы; прогон 16.11.2026, 08:17–10:16 UTC. +- `incident-signals.json`, `correlation.json` — реальные trace/log связи и диагнозы. +- `backend-outage.json` — ограниченная очередь, RAM и потери при отказе Tempo. +- `persistence-processes.json`, `failover-action.jsonl`, `pipeline.json` — реальные воздействия на процессы/volumes и 13 сценариев pipeline. +- `final-checks.txt`, `contracts.txt`, `build.txt`, `signal-checks.txt` — результаты проверок. +- `environment.json`, `image-provenance.json` — среда, fingerprint исходников и образы. +- `aborted-capacity-*`, `aborted-counter-*` — незавершённые промежуточные прогоны; они не считаются успешным исследованием. + +Методика, команды, результаты и ограничения: [REPORT.md](../REPORT.md). Даты измерений сохранены; итоговая проверка отчёта выполнена 17.11.2026. Секреты и локальные абсолютные пути не являются частью доказательств. diff --git a/evidence/aborted-capacity-diagnosis.md b/evidence/aborted-capacity-diagnosis.md new file mode 100644 index 0000000..764acbd --- /dev/null +++ b/evidence/aborted-capacity-diagnosis.md @@ -0,0 +1,16 @@ +# Незавершённый первый прогон + +2026-11-16 UTC. См. `aborted-capacity-slo-experiment.json` и `aborted-capacity-slo-run.txt`. + +За 996.6 s генератор получил 437 подтверждённых upload, 30 transport/timeouts и 31 ответ 429; после короткой остановки worker восстановление завершилось timeout запуска зависимого s3-init. Этот прогон не является доказательством успешно завершённого исследования. + +Наблюдавшийся native log SeaweedFS около 07:49 UTC: + +```text +No writable volumes and no free volumes left +create 7 volume, created 0: Not enough data nodes found! +``` + +В перенесённой конфигурации были `-volume.max=4` и `-master.volumeSizeLimitMB=128`. Ограничение роста writable volumes мешало записи данных/служебного журнала filer. API upload зависал до timeout, затем заполнялись upload slots и возвращался 429. Нулевой HTTP 5xx burn rate не означал отсутствие клиентских ошибок: timeout и 429 не входят в numerator серверного SLI. + +Исправлено без удаления volumes: volume.max=16, volumeSizeLimitMB=1024; worker/Tempo в эксперименте восстанавливаются прямым docker start существующего контейнера, без повторного запуска одноразовых зависимостей. После восстановления все существовавшие jobs были ready (487 на момент проверки), публичный smoke PASS. Затем основной эксперимент запущен заново с полным прогревом. diff --git a/evidence/aborted-capacity-incident-signals.json b/evidence/aborted-capacity-incident-signals.json new file mode 100644 index 0000000..e6c800b --- /dev/null +++ b/evidence/aborted-capacity-incident-signals.json @@ -0,0 +1,4 @@ +{ + "evidence": {}, + "collection_errors": [] +} diff --git a/evidence/aborted-capacity-slo-experiment.json b/evidence/aborted-capacity-slo-experiment.json new file mode 100644 index 0000000..0bf0623 --- /dev/null +++ b/evidence/aborted-capacity-slo-experiment.json @@ -0,0 +1,3444 @@ +{ + "started": 1794814265.052673, + "revision": "470d9d6f1e4fe4f594e37c2ba59550ee5ca54fc1", + "parameters": { + "warmup": 900, + "long": 240, + "cooldown": 660, + "repeats": 3, + "backend": 300, + "output": "evidence/slo-experiment.json" + }, + "profile": { + "message_rps": 10, + "read_fraction": 0.9, + "image_interval_seconds": 2, + "max_inflight": 64, + "transparent_retries": false + }, + "events": [ + { + "phase": "baseline", + "at": 1794814265.636649 + }, + { + "phase": "worker_short_1", + "at": 1794815165.995991 + } + ], + "observations": [ + { + "at": 1794814265.636999, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": null, + "ready_burn": 0.0 + }, + { + "at": 1794814272.780598, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814277.812505, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814282.8480299, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814287.881545, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814292.9169369, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814297.9481401, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814305.1576252, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814310.1965492, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814315.2303622, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814320.260522, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814325.299039, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814330.3320289, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814336.852701, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814341.8910348, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814346.9304268, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814351.975444, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814357.013744, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814362.045527, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814369.192048, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814374.221875, + "phase": "baseline", + "alerts": [], + "pending": 2.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814379.257214, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814384.293803, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814389.333649, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814394.375947, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814401.536777, + "phase": "baseline", + "alerts": [], + "pending": 2.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814406.5781279, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814411.6098392, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814416.653446, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814421.6958568, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814426.73916, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814433.927367, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814438.95975, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814443.9941251, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.446519, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814449.026679, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814454.053953, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814459.090545, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814465.839726, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814470.8650198, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814475.8960252, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814480.9348748, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814485.962085, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814490.995614, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814497.867564, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814502.9044201, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814507.944584, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814512.9781928, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814518.029838, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814523.083723, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814530.274218, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.048107, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814535.314759, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814540.346174, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814545.37646, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814550.411424, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814555.4566069, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814562.688919, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.055884, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814567.733532, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814572.77861, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814577.8304052, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814582.858988, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814587.8952441, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814595.123483, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814600.153224, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814605.197064, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814610.2418802, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814615.277948, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814620.307603, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814626.894887, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814631.934541, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814636.970549, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814642.007945, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814647.044236, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814652.090749, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814659.250651, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814664.285107, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814669.33301, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814674.36945, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814679.407016, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814684.445305, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814691.6184168, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814696.648727, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814701.682208, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814706.7110689, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814711.753027, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814716.794872, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814723.9788852, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814729.011874, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814734.044648, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814739.0969331, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814744.139233, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814749.193061, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814755.919819, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814760.95506, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814765.990944, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814771.027693, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814776.066644, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814781.0993989, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814787.9527, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814792.99052, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814798.039741, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814803.088059, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814808.124885, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814813.1576478, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814820.3259299, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814825.356671, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814830.418334, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814835.461644, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814840.508394, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814845.547137, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814852.7157419, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814857.753657, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814862.792468, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.049379, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814867.826991, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814872.903142, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814877.948224, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814885.131131, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814890.169672, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814895.2069728, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814900.2435699, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.012821, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814905.2937632, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814910.3328571, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814916.95196, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814921.998386, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814927.027981, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814932.070005, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814937.112621, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814942.151548, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814949.3282502, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814954.3685231, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814959.412212, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814964.4536982, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814969.4948559, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814974.530584, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814981.71894, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814986.7597618, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814991.797421, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794814996.830802, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815001.874923, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.034198, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815006.914997, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815014.0929668, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815019.1440718, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815024.177213, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815029.2042701, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815034.241496, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.055296, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815039.2928698, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815045.984266, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815051.02472, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815056.0598102, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815061.09712, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.065146, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815066.132726, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815071.18267, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815078.0082922, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815083.048585, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815088.095562, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815093.130262, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815098.169282, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815103.198426, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815110.3525162, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815115.384377, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815120.432491, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815125.465377, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815130.512164, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815135.545824, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815142.705439, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815147.7368279, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 1.112582, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815152.775208, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 6.246668, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815157.812719, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 11.37512, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815162.865692, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 16.487893, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815166.002381, + "phase": "worker_short_1", + "alerts": [], + "pending": 1.0, + "queue_age": 21.619195, + "api_burn": 0.0, + "ready_burn": 0.0 + } + ], + "resources": [ + { + "at": 1794814267.7754, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.03%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "27.58%", + "MemUsage": "141.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "659kB / 1.07MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.58%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "27.02%", + "MemUsage": "138.3MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "187kB / 565kB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.09%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "7.98%", + "MemUsage": "10.21MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "512kB / 1.35MB", + "PIDs": "8" + }, + { + "BlockIO": "4.61MB / 0B", + "CPUPerc": "0.45%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.22%", + "MemUsage": "85.05MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "11.8MB / 6.92MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.91%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "0.97%", + "MemUsage": "9.93MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "246kB / 157kB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814300.147837, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.73%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "27.66%", + "MemUsage": "141.6MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "1.89MB / 2.9MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.41%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "27.12%", + "MemUsage": "138.9MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "2.09MB / 3.09MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.74%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.10%", + "MemUsage": "10.36MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "676kB / 1.73MB", + "PIDs": "8" + }, + { + "BlockIO": "4.61MB / 0B", + "CPUPerc": "1.40%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.51%", + "MemUsage": "85.8MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "12.8MB / 7.39MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.59%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.02%", + "MemUsage": "10.46MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "2.21MB / 2.61MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814331.844568, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.68%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "27.75%", + "MemUsage": "142.1MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "4.07MB / 6.19MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.32%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "27.31%", + "MemUsage": "139.8MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "3.93MB / 5.94MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.77%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.12%", + "MemUsage": "10.4MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "849kB / 2.14MB", + "PIDs": "8" + }, + { + "BlockIO": "4.61MB / 0B", + "CPUPerc": "0.33%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.01%", + "MemUsage": "84.5MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "13.7MB / 7.83MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.18%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.05%", + "MemUsage": "10.73MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "4.17MB / 4.93MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814364.1886349, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.78%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "27.84%", + "MemUsage": "142.5MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "6.22MB / 9.5MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.92%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "27.30%", + "MemUsage": "139.8MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "5.94MB / 9.01MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.83%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.04%", + "MemUsage": "10.3MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "1.02MB / 2.53MB", + "PIDs": "8" + }, + { + "BlockIO": "4.61MB / 0B", + "CPUPerc": "0.34%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.75%", + "MemUsage": "86.41MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "14.7MB / 8.26MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.12%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.07%", + "MemUsage": "10.97MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "6.01MB / 7.12MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814396.5272698, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.03%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.40%", + "MemUsage": "17.39MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "8.43MB / 12.8MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.77%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.62%", + "MemUsage": "28.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "7.82MB / 12MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.92%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.13%", + "MemUsage": "10.41MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "1.18MB / 2.92MB", + "PIDs": "8" + }, + { + "BlockIO": "4.68MB / 0B", + "CPUPerc": "0.43%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.28%", + "MemUsage": "90.32MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "15.7MB / 8.7MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.59%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.09%", + "MemUsage": "11.11MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "8.09MB / 9.58MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814428.920022, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.54%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.39%", + "MemUsage": "17.36MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "10.6MB / 16.1MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.71%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.62%", + "MemUsage": "28.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "9.98MB / 15.4MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.62%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.14%", + "MemUsage": "10.42MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "1.35MB / 3.31MB", + "PIDs": "8" + }, + { + "BlockIO": "4.68MB / 0B", + "CPUPerc": "0.44%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.63%", + "MemUsage": "91.21MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "16.6MB / 9.1MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.76%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.07%", + "MemUsage": "10.95MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "10MB / 11.9MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814460.837277, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.44%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.40%", + "MemUsage": "17.4MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "13.1MB / 19.7MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.74%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.64%", + "MemUsage": "28.85MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "11.7MB / 18.2MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.80%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.17%", + "MemUsage": "10.45MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "1.52MB / 3.7MB", + "PIDs": "8" + }, + { + "BlockIO": "4.68MB / 0B", + "CPUPerc": "0.47%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.71%", + "MemUsage": "83.73MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "17.6MB / 9.49MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.24%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.11%", + "MemUsage": "11.35MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "12MB / 14.2MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814492.8656058, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.00%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.44%", + "MemUsage": "17.6MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "14.8MB / 22.5MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.01%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.59%", + "MemUsage": "28.62MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "14.1MB / 21.8MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.79%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.19%", + "MemUsage": "10.48MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "1.68MB / 4.09MB", + "PIDs": "8" + }, + { + "BlockIO": "4.81MB / 0B", + "CPUPerc": "0.52%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.90%", + "MemUsage": "86.79MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "18.5MB / 9.85MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.35%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.09%", + "MemUsage": "11.17MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "14MB / 16.6MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814525.260736, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.20%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.35%", + "MemUsage": "17.14MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "17.1MB / 26MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.69%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.62%", + "MemUsage": "28.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "16MB / 24.8MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.90%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.39%", + "MemUsage": "10.73MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "1.85MB / 4.49MB", + "PIDs": "8" + }, + { + "BlockIO": "4.81MB / 0B", + "CPUPerc": "0.90%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.13%", + "MemUsage": "84.8MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "19.5MB / 10.2MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.11%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.10%", + "MemUsage": "11.22MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "15.9MB / 18.9MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814557.681479, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.66%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.34%", + "MemUsage": "17.09MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "19.3MB / 29.4MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.85%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.64%", + "MemUsage": "28.88MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "18MB / 27.9MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.69%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.20%", + "MemUsage": "10.5MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "2.02MB / 4.89MB", + "PIDs": "8" + }, + { + "BlockIO": "4.81MB / 0B", + "CPUPerc": "0.36%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.29%", + "MemUsage": "85.23MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "20.4MB / 10.6MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.08%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.13%", + "MemUsage": "11.53MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "17.9MB / 21.2MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814590.115787, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.20%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.39%", + "MemUsage": "17.37MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "21.3MB / 32.5MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.93%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.65%", + "MemUsage": "28.92MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "20.2MB / 31.2MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.72%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.21%", + "MemUsage": "10.51MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "2.19MB / 5.29MB", + "PIDs": "8" + }, + { + "BlockIO": "4.81MB / 0B", + "CPUPerc": "1.47%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.63%", + "MemUsage": "83.52MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "21.4MB / 11MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.53%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.11%", + "MemUsage": "11.35MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "20MB / 23.7MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814621.884247, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.47%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.35%", + "MemUsage": "17.17MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "23.5MB / 35.8MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.72%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.63%", + "MemUsage": "28.84MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "22.3MB / 34.5MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.81%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.10%", + "MemUsage": "10.37MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "2.36MB / 5.67MB", + "PIDs": "8" + }, + { + "BlockIO": "4.81MB / 0B", + "CPUPerc": "0.27%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.81%", + "MemUsage": "86.55MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "22.3MB / 11.3MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.81%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.13%", + "MemUsage": "11.59MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "21.9MB / 26MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814654.242183, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.76%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.37%", + "MemUsage": "17.26MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "25.6MB / 39MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.28%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.65%", + "MemUsage": "28.92MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "24.3MB / 37.7MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.83%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.68%", + "MemUsage": "11.11MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "2.52MB / 6.07MB", + "PIDs": "8" + }, + { + "BlockIO": "4.94MB / 0B", + "CPUPerc": "0.82%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.23%", + "MemUsage": "87.62MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "23.3MB / 11.7MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.00%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.14%", + "MemUsage": "11.65MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "23.9MB / 28.3MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794814686.613148, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.04%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.38%", + "MemUsage": "17.29MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "27.8MB / 42.3MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.76%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.65%", + "MemUsage": "28.95MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "26.3MB / 40.9MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.89%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.20%", + "MemUsage": "10.5MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "2.69MB / 6.47MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.74%", + "MemUsage": "88.94MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "24.2MB / 12MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.76%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.14%", + "MemUsage": "11.72MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "25.8MB / 30.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794814718.971914, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.09%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.39%", + "MemUsage": "17.34MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "29.6MB / 45.2MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.99%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.67%", + "MemUsage": "29.02MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "28.7MB / 44.4MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.85%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.22%", + "MemUsage": "10.52MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "2.86MB / 6.86MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.69%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.05%", + "MemUsage": "89.73MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "25.2MB / 12.4MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.32%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.15%", + "MemUsage": "11.75MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "27.8MB / 32.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794814750.91762, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.72%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.36%", + "MemUsage": "17.21MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "31.6MB / 48.4MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.22%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.69%", + "MemUsage": "29.13MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "30.9MB / 47.7MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.86%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.26%", + "MemUsage": "10.57MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "3.03MB / 7.25MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.40%", + "MemUsage": "90.63MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "26.2MB / 12.8MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.23%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.15%", + "MemUsage": "11.77MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "29.7MB / 35.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794814782.945763, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.55%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.37%", + "MemUsage": "17.25MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "33.9MB / 51.8MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.48%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.70%", + "MemUsage": "29.17MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "32.8MB / 50.7MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.70%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.30%", + "MemUsage": "10.63MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "3.2MB / 7.64MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.31%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.54%", + "MemUsage": "85.86MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "27.1MB / 13.1MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.19%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.15%", + "MemUsage": "11.8MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "31.7MB / 37.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794814815.321418, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.38%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "3.43%", + "MemUsage": "17.59MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "36MB / 55.1MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.97%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.70%", + "MemUsage": "29.21MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "34.9MB / 53.8MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.04%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.37%", + "MemUsage": "10.71MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "3.36MB / 8.04MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "1.24%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.50%", + "MemUsage": "85.76MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "28.1MB / 13.5MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.13%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.13%", + "MemUsage": "11.54MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "33.7MB / 39.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794814847.703692, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.98%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "9.95%", + "MemUsage": "50.93MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "38.2MB / 58.5MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.55%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.74%", + "MemUsage": "29.41MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "36.9MB / 57.1MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.79%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.37%", + "MemUsage": "10.72MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "3.53MB / 8.44MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.46%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.71%", + "MemUsage": "86.3MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "29MB / 13.8MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.11%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.15%", + "MemUsage": "11.82MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "35.6MB / 42.2MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794814880.128491, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.53%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.43%", + "MemUsage": "68.76MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "40.3MB / 61.6MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.63%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.75%", + "MemUsage": "29.45MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "39.1MB / 60.4MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.78%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.24%", + "MemUsage": "10.54MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "3.7MB / 8.84MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.31%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.96%", + "MemUsage": "86.94MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "30MB / 14.2MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.53%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.16%", + "MemUsage": "11.88MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "37.7MB / 44.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794814911.9383419, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.22%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.62%", + "MemUsage": "69.73MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "42.2MB / 64.6MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.65%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.76%", + "MemUsage": "29.49MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "41.3MB / 63.9MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.76%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "8.39%", + "MemUsage": "10.74MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "3.87MB / 9.23MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.31%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.96%", + "MemUsage": "86.95MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "31MB / 14.6MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.61%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.17%", + "MemUsage": "12MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "39.7MB / 47MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794814944.314759, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.26%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.63%", + "MemUsage": "69.76MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "44.4MB / 67.9MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.93%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "5.77%", + "MemUsage": "29.54MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "43.3MB / 67MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.78%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.19%", + "MemUsage": "14.32MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "4.04MB / 9.62MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.96%", + "MemUsage": "86.95MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "31.9MB / 14.9MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.93%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.14%", + "MemUsage": "11.64MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "41.6MB / 49.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794814976.703645, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.86%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.62%", + "MemUsage": "69.75MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "46.4MB / 71.1MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.02%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "6.51%", + "MemUsage": "33.34MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "45.5MB / 70.3MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.97%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.20%", + "MemUsage": "14.34MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "4.2MB / 10MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.65%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.15%", + "MemUsage": "87.43MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "32.9MB / 15.3MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.81%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.17%", + "MemUsage": "11.99MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "43.6MB / 51.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794815009.080909, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.81%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.63%", + "MemUsage": "69.81MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "48.1MB / 74MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.99%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "13.83%", + "MemUsage": "70.8MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "48.1MB / 73.9MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.84%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.21%", + "MemUsage": "14.34MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "4.38MB / 10.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.58%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.28%", + "MemUsage": "87.75MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "33.8MB / 15.6MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.99%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.22%", + "MemUsage": "12.5MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "45.5MB / 54MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794815040.973404, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.82%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.65%", + "MemUsage": "69.9MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "50.1MB / 77.1MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.86%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "13.83%", + "MemUsage": "70.83MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "50.2MB / 77.2MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.72%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.22%", + "MemUsage": "14.36MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "4.54MB / 10.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.68%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.28%", + "MemUsage": "87.75MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "34.8MB / 16MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.17%", + "MemUsage": "12.03MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "47.5MB / 56.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794815073.004098, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.59%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.61%", + "MemUsage": "69.71MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "52.5MB / 80.6MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.53%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "13.98%", + "MemUsage": "71.55MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "51.9MB / 80.1MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.66%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.22%", + "MemUsage": "14.36MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "4.71MB / 11.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.36%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.28%", + "MemUsage": "87.75MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "35.7MB / 16.4MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.52%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.18%", + "MemUsage": "12.05MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "49.5MB / 58.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794815105.344744, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.10%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.60%", + "MemUsage": "69.64MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "54.4MB / 83.5MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.36%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "13.98%", + "MemUsage": "71.58MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "54.3MB / 83.7MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.80%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.22%", + "MemUsage": "14.37MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "4.88MB / 11.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "1.02%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.28%", + "MemUsage": "87.75MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "36.7MB / 16.7MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.83%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.50%", + "MemUsage": "15.34MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "51.4MB / 61MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794815137.697543, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.04%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.63%", + "MemUsage": "69.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "56.5MB / 86.8MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.96%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.05%", + "MemUsage": "71.93MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "56.5MB / 87MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.87%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.23%", + "MemUsage": "14.38MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "5.04MB / 12MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.39%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.04%", + "MemUsage": "87.13MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "37.6MB / 17.1MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.73%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.50%", + "MemUsage": "15.33MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "53.4MB / 63.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794815168.1513002, + "phase": "worker_short_1", + "containers": [ + { + "BlockIO": "0B / 459kB", + "CPUPerc": "1.43%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.68%", + "MemUsage": "70.04MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "58.3MB / 89.5MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 459kB", + "CPUPerc": "0.97%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.05%", + "MemUsage": "71.95MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "58.6MB / 89.5MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.79%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.24%", + "MemUsage": "14.39MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "5.17MB / 12.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.37%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.03%", + "MemUsage": "87.12MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "38.4MB / 17.4MB", + "PIDs": "19" + } + ] + } + ], + "phases": [ + { + "name": "baseline", + "start": 1794814265.636649, + "end": 1794815165.645, + "before": { + "requests": 0, + "success": 0, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 0, + "latency_success_ms": { + "0.5": null, + "0.95": null, + "0.99": null + }, + "statuses": {} + }, + "after": { + "requests": 9449, + "success": 9437, + "errors": 12, + "generator_dropped": 0, + "uploads_accepted": 437, + "latency_success_ms": { + "0.5": 8.547457866370678, + "0.95": 16.35137479752302, + "0.99": 25.318749714642763 + }, + "statuses": { + "0": 7, + "200": 8100, + "201": 900, + "202": 437, + "429": 5 + } + }, + "successful_rps": 10.485463388718314, + "error_ratio": 0.0012699756587998268, + "latency_success_ms": { + "0.5": 8.547457866370678, + "0.95": 16.35137479752302, + "0.99": 25.318749714642763 + }, + "statuses": { + "0": 7, + "200": 8100, + "201": 900, + "202": 437, + "429": 5 + } + }, + { + "name": "worker_short_1", + "start": 1794815165.995991, + "end": 1794815171.006418, + "before": { + "requests": 9454, + "success": 9441, + "errors": 13, + "generator_dropped": 0, + "uploads_accepted": 437, + "latency_success_ms": { + "0.5": 8.547457866370678, + "0.95": 16.35137479752302, + "0.99": 25.318749714642763 + }, + "statuses": { + "0": 7, + "200": 8103, + "201": 901, + "202": 437, + "429": 6 + } + }, + "after": { + "requests": 9507, + "success": 9491, + "errors": 16, + "generator_dropped": 0, + "uploads_accepted": 437, + "latency_success_ms": { + "0.5": 8.550749626010656, + "0.95": 16.34437497705221, + "0.99": 25.318749714642763 + }, + "statuses": { + "0": 8, + "200": 8148, + "201": 906, + "202": 437, + "429": 8 + } + }, + "successful_rps": 9.979205547217683, + "error_ratio": 0.05660377358490565, + "latency_success_ms": { + "0.5": 9.155082982033491, + "0.95": 14.380624983459711, + "0.99": 19.361665938049555 + }, + "statuses": { + "0": 1, + "200": 45, + "201": 5, + "429": 2 + } + } + ], + "image_id": "sha256:5157155d7b9bee87926b5d1cde95140a9118fce6e2b0d621e5e59c19bcf6d304", + "completed": false, + "failure_type": "TimeoutExpired", + "finished": 1794815261.653885, + "traffic": { + "requests": 10452, + "success": 10391, + "errors": 61, + "generator_dropped": 0, + "uploads_accepted": 437, + "latency_success_ms": { + "0.5": 8.605790790170431, + "0.95": 16.10241597518325, + "0.99": 25.203750003129244 + }, + "statuses": { + "0": 30, + "200": 8958, + "201": 996, + "202": 437, + "429": 31 + } + } +} diff --git a/evidence/aborted-capacity-slo-run.txt b/evidence/aborted-capacity-slo-run.txt new file mode 100644 index 0000000..840b280 --- /dev/null +++ b/evidence/aborted-capacity-slo-run.txt @@ -0,0 +1,44 @@ +baseline 900 seconds +baseline 2 s 24 requests +baseline 66 s 697 requests +baseline 131 s 1375 requests +baseline 195 s 2050 requests +baseline 260 s 2727 requests +baseline 324 s 3408 requests +baseline 389 s 4081 requests +baseline 453 s 4761 requests +baseline 517 s 5432 requests +baseline 582 s 6113 requests +baseline 646 s 6787 requests +baseline 711 s 7467 requests +baseline 775 s 8142 requests +baseline 840 s 8817 requests +worker_short_1 5 seconds +worker_short_1 2 s 9477 requests +Traceback (most recent call last): + File "/scripts/experiment-slo.py", line 143, in + if __name__=='__main__':main() + ^^^^^^ + File "/scripts/experiment-slo.py", line 113, in main + worker(5,f'worker_short_{i}');phase(f'worker_short_recovery_{i}',60) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scripts/experiment-slo.py", line 96, in worker + finally:compose('start','worker') + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scripts/experiment-slo.py", line 12, in compose + def compose(*a):return subprocess.run(C+list(a),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=90) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py", line 550, in run + stdout, stderr = process.communicate(input, timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py", line 1209, in communicate + stdout, stderr = self._communicate(input, endtime, timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py", line 2141, in _communicate + self.wait(timeout=self._remaining_time(endtime)) + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py", line 2045, in _wait + raise TimeoutExpired(self.args, timeout) +subprocess.TimeoutExpired: Command '['/scripts/compose.sh', 'start', 'worker']' timed out after 89.99991420796141 seconds diff --git a/evidence/aborted-counter-diagnosis.md b/evidence/aborted-counter-diagnosis.md new file mode 100644 index 0000000..36dd979 --- /dev/null +++ b/evidence/aborted-counter-diagnosis.md @@ -0,0 +1,15 @@ +# Промежуточная проверка учёта первого всплеска + +После исправления ёмкости S3 прогрев дал 9452 успешных ответа без ошибок. Короткая DB-блокировка дала 38 ответов 503 в генераторе. В native counters API было 30 business 5xx; часть остальных ответов создал LB при исчерпании DB pool и временной неготовности API. + +У новых 5xx time series первая точка Prometheus уже содержала весь короткий всплеск. Поэтому rate/increase показывали 0, несмотря на ненулевые raw counters. Этот прогон остановлен, а отсутствие alert не засчитано как успешное подавление короткого всплеска. + +Исправления: + +- При регистрации каждого маршрута HTTP counters публикуются с нулевыми значениями всех status classes. Первый всплеск имеет предшествующие нулевые samples. +- HTTP duration histogram объединяет status classes, сохраняя method/route: percentiles включают реальные медленные ошибки в уже существующих сериях без искусственных нулевых измерений. +- Jobs counters также инициализируются нулями известных outcomes. +- Добавлена gauge ожиданий Lock из pg_stat_activity, без SQL text/IDs; root cause можно отличить по сохранённым сигналам. +- DB pool в учебном профиле увеличен с 8 до 16 на процесс: при 10 message requests/s и timeout2s остаётся место для health. Это ограничивает побочное переключение LB, не снимая блокировку messages. + +Полное исследование затем запускается заново на исправленном образе. Промежуточные артефакты: `aborted-counter-*`. diff --git a/evidence/aborted-counter-incident-signals.json b/evidence/aborted-counter-incident-signals.json new file mode 100644 index 0000000..fa0e81b --- /dev/null +++ b/evidence/aborted-counter-incident-signals.json @@ -0,0 +1,62 @@ +{ + "evidence": { + "worker_long_1": { + "captured": 1794816601.2943192, + "observed": { + "at": 1794816600.582247, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "4.0476190476190477e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "6.3674393e+01" + } + ], + "pending": 32.0, + "queue_age": 63.674393, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + "worker_received_events": 0, + "accepted_upload_events": 35, + "diagnosis": "Queue age and ready burn increase while API accepts uploads and no worker consumption is observed" + } + }, + "collection_errors": [] +} diff --git a/evidence/aborted-counter-slo-experiment.json b/evidence/aborted-counter-slo-experiment.json new file mode 100644 index 0000000..a55c4f7 --- /dev/null +++ b/evidence/aborted-counter-slo-experiment.json @@ -0,0 +1,6121 @@ +{ + "started": 1794815497.204417, + "revision": "470d9d6f1e4fe4f594e37c2ba59550ee5ca54fc1", + "parameters": { + "warmup": 900, + "long": 240, + "cooldown": 660, + "repeats": 3, + "backend": 300, + "output": "evidence/slo-experiment.json" + }, + "profile": { + "message_rps": 10, + "read_fraction": 0.9, + "image_interval_seconds": 2, + "max_inflight": 64, + "transparent_retries": false + }, + "events": [ + { + "phase": "baseline", + "at": 1794815498.040816 + }, + { + "phase": "worker_short_1", + "at": 1794816398.391487 + }, + { + "phase": "worker_short_recovery_1", + "at": 1794816403.613207 + }, + { + "phase": "db_short_1", + "at": 1794816463.762387 + }, + { + "phase": "db_short_recovery_1", + "at": 1794816468.896548 + }, + { + "phase": "worker_long_1", + "at": 1794816529.292293 + } + ], + "observations": [ + { + "at": 1794815498.0413709, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815505.2339141, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815510.264962, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815515.2915268, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815520.340658, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815525.384007, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815530.4332721, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815537.0751529, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815542.124362, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.041203, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815547.163737, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815552.201822, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815557.230619, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815562.2746089, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815569.443904, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815574.48861, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815579.5239239, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815584.564169, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815589.592289, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815594.6284978, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815601.805553, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.07037, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815606.842647, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815611.886785, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815616.926604, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815621.9596531, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815626.993729, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815634.165441, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.094979, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815639.193878, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815644.2238662, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815649.257904, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815654.298559, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815659.33803, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815666.064266, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815671.098633, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815676.133303, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815681.169194, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815686.2018921, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815691.235345, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815698.096703, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815703.129823, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815708.167168, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815713.19681, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815718.2510982, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815723.284163, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815730.4608629, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.049144, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815735.499668, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815740.534506, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815745.568083, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815750.596364, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815755.637717, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815762.795375, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.063352, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815767.8334908, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815772.8648179, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815777.906292, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815782.9444091, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815787.982432, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815795.2356951, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815800.276703, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815805.3179321, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815810.373895, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815815.415307, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815820.459015, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815827.125728, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815832.170027, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815837.196728, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815842.2356749, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815847.265037, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815852.3058329, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815859.478815, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815864.5123892, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815869.559508, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815874.600254, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815879.64874, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815884.681613, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815891.8486998, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815896.836321, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815901.875836, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815906.913249, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815911.957182, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815917.008147, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815924.197855, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815929.241455, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815934.284192, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815939.318593, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815944.348059, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815949.384429, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815956.098408, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815961.132837, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815966.165174, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815971.194864, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815976.229311, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815981.257772, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.06274, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815988.075831, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815993.11565, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794815998.148387, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816003.189363, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816008.245253, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816013.278486, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.002618, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816020.4764428, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816025.528219, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816030.582357, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816035.6382048, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816040.670835, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.10167, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816045.710824, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816052.899921, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816057.9360058, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816062.984565, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816068.0401769, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816073.087494, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.057751, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816078.1214669, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816085.313162, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816090.387777, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816095.41443, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816100.444423, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816105.4969091, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816110.564981, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816117.11806, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816122.16365, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "queue_age": 0.007564, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816127.199616, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816132.238158, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816137.266252, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816142.3024871, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816149.461131, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816154.496664, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816159.534327, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816164.583584, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816169.6320992, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816174.684984, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816181.856048, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816186.9089642, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816191.9425979, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816196.975778, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816202.012558, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816207.057289, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816214.2278159, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816219.275139, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816224.3027892, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816229.331255, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816234.368136, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816239.407846, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816246.1112819, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816251.140204, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816256.174657, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816261.215679, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816266.245131, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816271.278644, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816278.146127, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816283.182474, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816288.233355, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816293.266942, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816298.314763, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816303.3416471, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816310.520689, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816315.558501, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816320.591083, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816325.638023, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816330.67414, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816335.726034, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816342.8908749, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816347.945354, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816352.982691, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816358.017781, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816363.059056, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816368.108133, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816375.302001, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816380.349975, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816385.390999, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816390.424557, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816395.455093, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816398.399424, + "phase": "worker_short_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816403.619104, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816410.793579, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816415.826096, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816420.8552918, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816425.908629, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816430.97475, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816436.02798, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816443.185149, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816448.2338371, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816453.274674, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816458.305523, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816463.349608, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816463.768585, + "phase": "db_short_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816468.903035, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816476.116679, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816481.147136, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816486.194318, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816491.223613, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816496.256381, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816501.298723, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816508.163221, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816513.1995032, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816518.234921, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816523.2659981, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816528.305872, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816529.300142, + "phase": "worker_long_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816536.180879, + "phase": "worker_long_1", + "alerts": [], + "pending": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816541.21634, + "phase": "worker_long_1", + "alerts": [], + "pending": 2.0, + "queue_age": 3.892575, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816546.2550359, + "phase": "worker_long_1", + "alerts": [], + "pending": 5.0, + "queue_age": 9.058895, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816551.286318, + "phase": "worker_long_1", + "alerts": [], + "pending": 8.0, + "queue_age": 14.258335, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816556.328296, + "phase": "worker_long_1", + "alerts": [], + "pending": 10.0, + "queue_age": 19.390146, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816561.366697, + "phase": "worker_long_1", + "alerts": [], + "pending": 13.0, + "queue_age": 24.533713, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816568.211828, + "phase": "worker_long_1", + "alerts": [], + "pending": 15.0, + "queue_age": 28.700176, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794816573.24789, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "3.3826902e+01" + } + ], + "pending": 17.0, + "queue_age": 33.826902, + "api_burn": 0.0, + "ready_burn": 16.666666666666664 + }, + { + "at": 1794816578.291688, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "3.896825e+01" + } + ], + "pending": 20.0, + "queue_age": 38.96825, + "api_burn": 0.0, + "ready_burn": 41.666666666666664 + }, + { + "at": 1794816583.327447, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "6.153846153846154e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "4.4081287e+01" + } + ], + "pending": 23.0, + "queue_age": 44.081287, + "api_burn": 0.0, + "ready_burn": 61.53846153846154 + }, + { + "at": 1794816588.363501, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "7.692307692307693e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "2.3255813953488372e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "4.9204616e+01" + } + ], + "pending": 25.0, + "queue_age": 49.204616, + "api_burn": 0.0, + "ready_burn": 76.92307692307692 + }, + { + "at": 1794816593.407267, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "3.023255813953489e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "5.4404638e+01" + } + ], + "pending": 28.0, + "queue_age": 54.404638, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816600.582247, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "4.0476190476190477e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "6.3674393e+01" + } + ], + "pending": 32.0, + "queue_age": 63.674393, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816605.625867, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "4.761904761904762e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "6.8876896e+01" + } + ], + "pending": 35.0, + "queue_age": 68.876896, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816610.681428, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "5.238095238095238e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "7.3994943e+01" + } + ], + "pending": 37.0, + "queue_age": 73.994943, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816615.709672, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "5.813953488372092e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "7.918995e+01" + } + ], + "pending": 40.0, + "queue_age": 79.18995, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816620.749679, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "6.511627906976745e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "8.4333016e+01" + } + ], + "pending": 43.0, + "queue_age": 84.333016, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816625.7930639, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "6.976744186046512e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "8.9459252e+01" + } + ], + "pending": 45.0, + "queue_age": 89.459252, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816632.963167, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "7.61904761904762e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "9.3555765e+01" + } + ], + "pending": 47.0, + "queue_age": 93.555765, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816637.998452, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "8.333333333333333e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "9.8682343e+01" + } + ], + "pending": 50.0, + "queue_age": 98.682343, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816643.06992, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "8.809523809523808e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.03816649e+02" + } + ], + "pending": 52.0, + "queue_age": 103.816649, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816648.120556, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "9.302325581395349e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.09045214e+02" + } + ], + "pending": 55.0, + "queue_age": 109.045214, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816653.1628711, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.14255356e+02" + } + ], + "pending": 58.0, + "queue_age": 114.255356, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816658.204293, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.19388791e+02" + } + ], + "pending": 60.0, + "queue_age": 119.388791, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816665.409659, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.28619006e+02" + } + ], + "pending": 65.0, + "queue_age": 128.619006, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816670.440489, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.33817277e+02" + } + ], + "pending": 67.0, + "queue_age": 133.817277, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816675.4920251, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.38952996e+02" + } + ], + "pending": 70.0, + "queue_age": 138.952996, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816680.536127, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.44074491e+02" + } + ], + "pending": 73.0, + "queue_age": 144.074491, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816685.58196, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.49203039e+02" + } + ], + "pending": 75.0, + "queue_age": 149.203039, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816690.612291, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.54346858e+02" + } + ], + "pending": 78.0, + "queue_age": 154.346858, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816697.2216399, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.59547962e+02" + } + ], + "pending": 80.0, + "queue_age": 159.547962, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816702.274, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.6365609e+02" + } + ], + "pending": 82.0, + "queue_age": 163.65609, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816707.309789, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.68873504e+02" + } + ], + "pending": 85.0, + "queue_age": 168.873504, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816712.33894, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.74086091e+02" + } + ], + "pending": 88.0, + "queue_age": 174.086091, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816717.3870442, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.79228577e+02" + } + ], + "pending": 90.0, + "queue_age": 179.228577, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816722.435414, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.84429505e+02" + } + ], + "pending": 93.0, + "queue_age": 184.429505, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816728.604725, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.89570034e+02" + } + ], + "pending": 95.0, + "queue_age": 189.570034, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816733.6454248, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.93686648e+02" + } + ], + "pending": 97.0, + "queue_age": 193.686648, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794816738.7131252, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:39.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:44.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:09:29.758725496Z", + "value": "1.98882337e+02" + } + ], + "pending": 100.0, + "queue_age": 198.882337, + "api_burn": 0.0, + "ready_burn": 100.0 + } + ], + "resources": [ + { + "at": 1794815500.231321, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.78%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "51.17%", + "MemUsage": "262MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "64.6MB / 99MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.67%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "51.59%", + "MemUsage": "264.1MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "65.5MB / 99.3MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.75%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.28%", + "MemUsage": "14.44MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "6.5MB / 15.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "5.88%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.47%", + "MemUsage": "83.13MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "44MB / 19.8MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "3.32%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.32%", + "MemUsage": "13.55MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "550kB / 773kB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815532.0649679, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.36%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "52.69%", + "MemUsage": "269.8MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "66.4MB / 101MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.03%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "52.09%", + "MemUsage": "266.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "66.8MB / 101MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.90%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.31%", + "MemUsage": "14.48MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "6.67MB / 16.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.85%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.48%", + "MemUsage": "83.15MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "45.1MB / 20.1MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.23%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.34%", + "MemUsage": "13.67MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "2.39MB / 2.97MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815564.437598, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.38%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "52.74%", + "MemUsage": "270.1MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "68.5MB / 105MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.68%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "52.48%", + "MemUsage": "268.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "68.8MB / 104MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.85%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.30%", + "MemUsage": "14.47MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "6.84MB / 16.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "1.09%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.53%", + "MemUsage": "83.29MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "46.1MB / 20.5MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.25%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.38%", + "MemUsage": "14.09MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "4.35MB / 5.3MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815596.798264, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.94%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "53.91%", + "MemUsage": "276MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "70.6MB / 108MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "2.02%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "53.27%", + "MemUsage": "272.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "70.7MB / 107MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.83%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.60%", + "MemUsage": "14.85MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "7.01MB / 17.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.79%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.54%", + "MemUsage": "83.3MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "47.2MB / 20.9MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.16%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.38%", + "MemUsage": "14.16MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "6.31MB / 7.62MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815629.155322, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.65%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.90%", + "MemUsage": "71.16MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "72.7MB / 111MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.84%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.81%", + "MemUsage": "75.84MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "72.9MB / 111MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.79%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.48%", + "MemUsage": "14.7MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "7.18MB / 17.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.58%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.71%", + "MemUsage": "86.29MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "48.3MB / 21.3MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "3.52%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.42%", + "MemUsage": "14.51MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "8.39MB / 10.1MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815661.061019, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.60%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.89%", + "MemUsage": "71.09MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "74.8MB / 115MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.07%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.69%", + "MemUsage": "75.22MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "75MB / 114MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.81%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.62%", + "MemUsage": "14.87MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "7.35MB / 17.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.35%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.87%", + "MemUsage": "86.7MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "49.4MB / 21.7MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "3.14%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.43%", + "MemUsage": "14.62MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "10.3MB / 12.4MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815693.084019, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.05%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.77%", + "MemUsage": "70.52MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "76.8MB / 118MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.15%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.69%", + "MemUsage": "75.21MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "77.2MB / 117MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.89%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.61%", + "MemUsage": "14.86MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "7.51MB / 18.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.03MB / 0B", + "CPUPerc": "0.55%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.96%", + "MemUsage": "86.95MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "50.4MB / 22MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "3.54%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.43%", + "MemUsage": "14.62MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "12.3MB / 14.8MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815725.457919, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.85%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.78%", + "MemUsage": "70.54MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "79MB / 121MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.99%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.62%", + "MemUsage": "74.86MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "79.2MB / 121MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.76%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.71%", + "MemUsage": "14.98MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "7.68MB / 18.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "0.70%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.98%", + "MemUsage": "86.98MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "51.5MB / 22.4MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.48%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.43%", + "MemUsage": "14.66MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "14.3MB / 17.1MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815757.792542, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.54%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.81%", + "MemUsage": "70.72MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "81.1MB / 125MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.81%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.63%", + "MemUsage": "74.92MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "81.3MB / 124MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.66%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.66%", + "MemUsage": "14.93MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "7.85MB / 19MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "5.74%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.24%", + "MemUsage": "87.66MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "52.6MB / 22.8MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "3.69%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.45%", + "MemUsage": "14.84MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "16.2MB / 19.4MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815790.232168, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "2.67%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.83%", + "MemUsage": "70.79MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "83.4MB / 128MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.95%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.52%", + "MemUsage": "74.33MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "83.3MB / 127MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.10%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.68%", + "MemUsage": "14.95MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "8.02MB / 19.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "2.05%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.84%", + "MemUsage": "84.06MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "53.7MB / 23.2MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "1.45%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.45%", + "MemUsage": "14.87MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "18.2MB / 21.7MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815822.115078, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.95%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.83%", + "MemUsage": "70.82MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "85.6MB / 131MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.98%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.62%", + "MemUsage": "74.87MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "85.2MB / 130MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.85%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.68%", + "MemUsage": "14.95MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "8.18MB / 19.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "0.94%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.34%", + "MemUsage": "85.36MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "54.8MB / 23.6MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "1.22%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.46%", + "MemUsage": "14.93MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "20.1MB / 24.1MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815854.470753, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "2.46%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.82%", + "MemUsage": "70.78MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "87.9MB / 135MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.14%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.64%", + "MemUsage": "74.96MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "87.2MB / 133MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.69%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.69%", + "MemUsage": "14.96MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "8.34MB / 20.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "1.04%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.19%", + "MemUsage": "82.42MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "55.8MB / 24MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.21%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.46%", + "MemUsage": "14.97MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "22.1MB / 26.4MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815886.834761, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "2.10%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.80%", + "MemUsage": "70.64MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "89.7MB / 138MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "2.73%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.70%", + "MemUsage": "75.26MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "89.5MB / 137MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.90%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.87%", + "MemUsage": "15.2MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "8.52MB / 20.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "0.39%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.03%", + "MemUsage": "81.99MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "56.9MB / 24.3MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.15%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.47%", + "MemUsage": "15.02MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "24.1MB / 28.7MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815919.1907809, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.00%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.82%", + "MemUsage": "70.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "92.1MB / 141MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.96%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.67%", + "MemUsage": "75.1MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "91.3MB / 140MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.78%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.79%", + "MemUsage": "15.09MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "8.68MB / 21MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "0.37%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.40%", + "MemUsage": "85.5MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "58MB / 24.7MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.26%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.47%", + "MemUsage": "15.06MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "26.1MB / 31.2MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815951.0933251, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.68%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.73%", + "MemUsage": "70.32MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "94.1MB / 145MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.94%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.64%", + "MemUsage": "74.97MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "93.5MB / 143MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.73%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.87%", + "MemUsage": "15.19MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "8.85MB / 21.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "0.50%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.85%", + "MemUsage": "84.09MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "59.1MB / 25.1MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "3.55%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.48%", + "MemUsage": "15.11MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "28.1MB / 33.5MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794815983.072327, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.93%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.82%", + "MemUsage": "70.75MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "96.4MB / 148MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.09%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.65%", + "MemUsage": "75MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "95.4MB / 146MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.25%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.82%", + "MemUsage": "15.13MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "9.02MB / 21.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "0.57%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.13%", + "MemUsage": "84.8MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "60.1MB / 25.5MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "3.65%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.48%", + "MemUsage": "15.14MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "30.1MB / 35.9MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794816015.462694, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.95%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.82%", + "MemUsage": "70.78MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "98.4MB / 151MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.11%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.68%", + "MemUsage": "75.16MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "97.6MB / 150MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.64%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.82%", + "MemUsage": "15.13MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "9.19MB / 22.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "1.01%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.03%", + "MemUsage": "84.56MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "61.2MB / 25.9MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.19%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.48%", + "MemUsage": "15.18MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "32MB / 38.2MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794816047.889259, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "2.12%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.84%", + "MemUsage": "70.86MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "101MB / 155MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "2.62%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.60%", + "MemUsage": "74.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "99.2MB / 152MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.07%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "12.06%", + "MemUsage": "15.44MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "9.35MB / 22.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "0.60%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.42%", + "MemUsage": "85.55MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "62.3MB / 26.3MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.14%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.49%", + "MemUsage": "15.22MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "34MB / 40.5MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794816080.3051672, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.70%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.90%", + "MemUsage": "71.15MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "103MB / 158MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.39%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.66%", + "MemUsage": "75.08MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "101MB / 156MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.08%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.87%", + "MemUsage": "15.19MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "9.53MB / 23MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "1.32%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.02%", + "MemUsage": "87.09MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "63.4MB / 26.7MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "3.25%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.49%", + "MemUsage": "15.25MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "36.1MB / 43MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794816112.114066, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.11%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.81%", + "MemUsage": "70.68MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "105MB / 162MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.79%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.69%", + "MemUsage": "75.21MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "104MB / 159MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.78%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.93%", + "MemUsage": "15.27MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "9.69MB / 23.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "0.34%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.89%", + "MemUsage": "84.2MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "64.5MB / 27MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "2.64%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.78%", + "MemUsage": "18.2MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "38MB / 45.2MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794816144.451166, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "2.11%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.81%", + "MemUsage": "70.69MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "107MB / 165MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.97%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.70%", + "MemUsage": "75.26MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "106MB / 163MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.04%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.94%", + "MemUsage": "15.29MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "9.86MB / 23.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "0.42%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.77%", + "MemUsage": "83.89MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "65.5MB / 27.4MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.22%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.49%", + "MemUsage": "15.29MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "39.8MB / 47.5MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794816176.843466, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.61%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.84%", + "MemUsage": "70.88MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "109MB / 168MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.55%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.66%", + "MemUsage": "75.04MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "108MB / 166MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.05%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.95%", + "MemUsage": "15.3MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "10MB / 24.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "0.49%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.02%", + "MemUsage": "84.54MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "66.6MB / 27.8MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "3.70%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.50%", + "MemUsage": "15.33MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "41.9MB / 50MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794816209.220381, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.63%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.85%", + "MemUsage": "70.91MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "111MB / 171MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.80%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.68%", + "MemUsage": "75.14MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "110MB / 169MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.81%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.86%", + "MemUsage": "15.18MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "10.2MB / 24.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.16MB / 0B", + "CPUPerc": "0.65%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.71%", + "MemUsage": "88.87MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "67.7MB / 28.2MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.21%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.70%", + "MemUsage": "17.39MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "43.9MB / 52.3MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794816241.107285, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.62%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.85%", + "MemUsage": "70.93MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "113MB / 174MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.69%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.74%", + "MemUsage": "75.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "113MB / 173MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.75%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.87%", + "MemUsage": "15.19MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "10.4MB / 24.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.27MB / 0B", + "CPUPerc": "0.42%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.76%", + "MemUsage": "88.98MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "68.8MB / 28.6MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "3.48%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.71%", + "MemUsage": "17.5MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "45.8MB / 54.6MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794816273.138477, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.91%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.85%", + "MemUsage": "70.94MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "114MB / 177MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.93%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.71%", + "MemUsage": "75.29MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "115MB / 177MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.94%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.87%", + "MemUsage": "15.19MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "10.5MB / 25.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.27MB / 0B", + "CPUPerc": "0.53%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.95%", + "MemUsage": "89.48MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "69.8MB / 28.9MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.27%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.71%", + "MemUsage": "17.52MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "47.8MB / 57MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794816305.510589, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.03%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.86%", + "MemUsage": "70.95MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "116MB / 180MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.94%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.73%", + "MemUsage": "75.44MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "117MB / 180MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.88%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.89%", + "MemUsage": "15.22MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "10.7MB / 25.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.27MB / 0B", + "CPUPerc": "2.49%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.93%", + "MemUsage": "89.41MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "70.9MB / 29.3MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.17%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.71%", + "MemUsage": "17.54MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "49.8MB / 59.3MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794816337.872185, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.47%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.86%", + "MemUsage": "70.96MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "119MB / 184MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.91%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.67%", + "MemUsage": "75.12MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "119MB / 183MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.01%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.87%", + "MemUsage": "15.2MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "10.9MB / 26.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.27MB / 0B", + "CPUPerc": "0.52%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.51%", + "MemUsage": "88.36MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "72MB / 29.7MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "3.77%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.71%", + "MemUsage": "17.56MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "51.7MB / 61.6MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794816370.291979, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.73%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.86%", + "MemUsage": "70.98MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "121MB / 187MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.60%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.67%", + "MemUsage": "75.09MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "121MB / 186MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.75%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.89%", + "MemUsage": "15.21MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11MB / 26.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.27MB / 0B", + "CPUPerc": "0.90%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.52%", + "MemUsage": "88.36MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "73.1MB / 30.1MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 135kB", + "CPUPerc": "0.22%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.72%", + "MemUsage": "17.58MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "53.7MB / 63.9MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794816400.601563, + "phase": "worker_short_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.61%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.80%", + "MemUsage": "70.64MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "123MB / 191MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "2.16%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.68%", + "MemUsage": "75.18MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "123MB / 189MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.70%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.89%", + "MemUsage": "15.22MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.2MB / 26.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.27MB / 0B", + "CPUPerc": "0.66%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "31.78%", + "MemUsage": "81.36MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "74.1MB / 30.4MB", + "PIDs": "19" + } + ] + }, + { + "at": 1794816405.780671, + "phase": "worker_short_recovery_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.88%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.80%", + "MemUsage": "70.64MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "123MB / 191MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.97%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.69%", + "MemUsage": "75.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "123MB / 190MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.04%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.90%", + "MemUsage": "15.23MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.2MB / 27MB", + "PIDs": "8" + }, + { + "BlockIO": "5.27MB / 0B", + "CPUPerc": "0.70%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.03%", + "MemUsage": "82MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "74.3MB / 30.5MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.13%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.29%", + "MemUsage": "13.24MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "378kB / 438kB", + "PIDs": "9" + } + ] + }, + { + "at": 1794816438.1782668, + "phase": "worker_short_recovery_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.34%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.85%", + "MemUsage": "70.89MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "126MB / 194MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.81%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.71%", + "MemUsage": "75.32MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "125MB / 193MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.95%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "12.13%", + "MemUsage": "15.52MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.4MB / 27.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.27MB / 0B", + "CPUPerc": "0.80%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.37%", + "MemUsage": "85.42MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "75.4MB / 30.9MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.16%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.34%", + "MemUsage": "13.75MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "2.34MB / 2.76MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794816465.904465, + "phase": "db_short_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.94%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.88%", + "MemUsage": "71.07MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "128MB / 197MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.09%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.71%", + "MemUsage": "75.34MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "127MB / 195MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.66%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "12.03%", + "MemUsage": "15.4MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.5MB / 27.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.27MB / 0B", + "CPUPerc": "0.32%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.52%", + "MemUsage": "88.37MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "76.3MB / 31.2MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.46%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.35%", + "MemUsage": "13.86MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "4.05MB / 4.79MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794816471.1072278, + "phase": "db_short_recovery_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.61%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.97%", + "MemUsage": "71.54MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "128MB / 198MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.63%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.84%", + "MemUsage": "75.99MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "127MB / 196MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.01%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "12.04%", + "MemUsage": "15.41MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.6MB / 27.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.38MB / 0B", + "CPUPerc": "1.29%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.58%", + "MemUsage": "88.52MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "76.4MB / 31.3MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.87%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.36%", + "MemUsage": "13.88MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "4.42MB / 5.23MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794816503.1584158, + "phase": "db_short_recovery_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.64%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.78%", + "MemUsage": "70.56MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "131MB / 202MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.59%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.69%", + "MemUsage": "75.23MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "129MB / 199MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.73%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.85%", + "MemUsage": "15.16MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.7MB / 28.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.38MB / 0B", + "CPUPerc": "0.35%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.62%", + "MemUsage": "88.63MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "77.5MB / 31.7MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.18%", + "Container": "52d33050ce3e7540d0eedff601e83d5df533a8e7883ce774bd5e1335c3e238f5", + "ID": "52d33050ce3e", + "MemPerc": "1.40%", + "MemUsage": "14.39MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "6.38MB / 7.55MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794816531.1627278, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.88%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.81%", + "MemUsage": "70.73MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "133MB / 205MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.02%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.69%", + "MemUsage": "75.24MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "131MB / 201MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.86%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.85%", + "MemUsage": "15.17MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.9MB / 28.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.38MB / 0B", + "CPUPerc": "0.56%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.94%", + "MemUsage": "89.45MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "78.5MB / 32MB", + "PIDs": "19" + } + ] + }, + { + "at": 1794816563.207031, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.55%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.81%", + "MemUsage": "70.69MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "134MB / 208MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.60%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.72%", + "MemUsage": "75.35MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "133MB / 205MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.47%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "12.11%", + "MemUsage": "15.5MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "12.1MB / 28.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.38MB / 0B", + "CPUPerc": "0.33%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.95%", + "MemUsage": "89.46MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "79.5MB / 32.4MB", + "PIDs": "19" + } + ] + }, + { + "at": 1794816595.571678, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.74%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.87%", + "MemUsage": "71MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "136MB / 211MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.02%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.74%", + "MemUsage": "75.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "135MB / 209MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.78%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.91%", + "MemUsage": "15.24MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "12.3MB / 29.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.38MB / 0B", + "CPUPerc": "0.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.67%", + "MemUsage": "88.76MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "80.4MB / 32.7MB", + "PIDs": "19" + } + ] + }, + { + "at": 1794816627.957341, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.80%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.87%", + "MemUsage": "71.04MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "138MB / 214MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.63%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.80%", + "MemUsage": "75.76MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "138MB / 212MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.93%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.81%", + "MemUsage": "15.12MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "12.4MB / 29.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.38MB / 0B", + "CPUPerc": "0.34%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.55%", + "MemUsage": "88.46MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "81.4MB / 33MB", + "PIDs": "19" + } + ] + }, + { + "at": 1794816660.395668, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "2.24%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.88%", + "MemUsage": "71.09MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "140MB / 217MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "1.70%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.76%", + "MemUsage": "75.55MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "140MB / 216MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.79%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.82%", + "MemUsage": "15.12MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "12.6MB / 30.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.38MB / 0B", + "CPUPerc": "1.14%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.47%", + "MemUsage": "93.37MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "82.4MB / 33.4MB", + "PIDs": "19" + } + ] + }, + { + "at": 1794816692.2180219, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "1.12%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.99%", + "MemUsage": "71.62MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "142MB / 220MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "2.15%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.75%", + "MemUsage": "75.5MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "142MB / 219MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.92%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.82%", + "MemUsage": "15.13MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "12.8MB / 30.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.38MB / 0B", + "CPUPerc": "0.48%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.47%", + "MemUsage": "95.93MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "83.3MB / 33.7MB", + "PIDs": "19" + } + ] + }, + { + "at": 1794816723.590066, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 1.61MB", + "CPUPerc": "0.86%", + "Container": "b9de3e0d130c0dc57b957d5a5349d4d1638aedda252f973de6fc83d0a33ce53d", + "ID": "b9de3e0d130c", + "MemPerc": "13.90%", + "MemUsage": "71.14MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "144MB / 224MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 1.84MB", + "CPUPerc": "0.72%", + "Container": "c775fb4331a25a53ecc59547263d37770ce99c300d1662a6655f57c4a873afc4", + "ID": "c775fb4331a2", + "MemPerc": "14.75%", + "MemUsage": "75.5MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "144MB / 222MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.52%", + "Container": "79c749e2aef2346e005bde233cfcbd7b9ce5828065ee2d7abab3e50e058d59c8", + "ID": "79c749e2aef2", + "MemPerc": "11.82%", + "MemUsage": "15.13MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "13MB / 30.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.46MB / 0B", + "CPUPerc": "0.53%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.26%", + "MemUsage": "92.82MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "84.2MB / 34.1MB", + "PIDs": "19" + } + ] + } + ], + "phases": [ + { + "name": "baseline", + "start": 1794815498.040816, + "end": 1794816398.042573, + "before": { + "requests": 0, + "success": 0, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 0, + "latency_success_ms": { + "0.5": null, + "0.95": null, + "0.99": null + }, + "statuses": {} + }, + "after": { + "requests": 9452, + "success": 9452, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 451, + "latency_success_ms": { + "0.5": 8.416832890361547, + "0.95": 16.48354111239314, + "0.99": 28.727833181619644 + }, + "statuses": { + "200": 8100, + "201": 901, + "202": 451 + } + }, + "successful_rps": 10.50220565461194, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.416832890361547, + "0.95": 16.48354111239314, + "0.99": 28.727833181619644 + }, + "statuses": { + "200": 8100, + "201": 901, + "202": 451 + } + }, + { + "name": "worker_short_1", + "start": 1794816398.391487, + "end": 1794816403.399065, + "before": { + "requests": 9455, + "success": 9455, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 451, + "latency_success_ms": { + "0.5": 8.417165838181973, + "0.95": 16.48354111239314, + "0.99": 28.727833181619644 + }, + "statuses": { + "200": 8103, + "201": 901, + "202": 451 + } + }, + "after": { + "requests": 9507, + "success": 9507, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 453, + "latency_success_ms": { + "0.5": 8.408665657043457, + "0.95": 16.496040858328342, + "0.99": 28.727833181619644 + }, + "statuses": { + "200": 8148, + "201": 906, + "202": 453 + } + }, + "successful_rps": 10.3842736943944, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 7.6812501065433025, + "0.95": 19.36087477952242, + "0.99": 34.383624792099 + }, + "statuses": { + "200": 45, + "201": 5, + "202": 2 + } + }, + { + "name": "worker_short_recovery_1", + "start": 1794816403.613207, + "end": 1794816463.617981, + "before": { + "requests": 9510, + "success": 9510, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 453, + "latency_success_ms": { + "0.5": 8.408665657043457, + "0.95": 16.524207778275013, + "0.99": 28.727833181619644 + }, + "statuses": { + "200": 8151, + "201": 906, + "202": 453 + } + }, + "after": { + "requests": 10140, + "success": 10140, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 483, + "latency_success_ms": { + "0.5": 8.420666214078665, + "0.95": 16.477208118885756, + "0.99": 28.588749933987856 + }, + "statuses": { + "200": 8691, + "201": 966, + "202": 483 + } + }, + "successful_rps": 10.499170732432013, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.616332896053791, + "0.95": 16.236250288784504, + "0.99": 27.301250025629997 + }, + "statuses": { + "200": 540, + "201": 60, + "202": 30 + } + }, + { + "name": "db_short_1", + "start": 1794816463.762387, + "end": 1794816468.7638242, + "before": { + "requests": 10141, + "success": 10141, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 483, + "latency_success_ms": { + "0.5": 8.421165868639946, + "0.95": 16.477208118885756, + "0.99": 28.588749933987856 + }, + "statuses": { + "200": 8692, + "201": 966, + "202": 483 + } + }, + "after": { + "requests": 10194, + "success": 10156, + "errors": 38, + "generator_dropped": 0, + "uploads_accepted": 486, + "latency_success_ms": { + "0.5": 8.436957839876413, + "0.95": 16.664416063576937, + "0.99": 29.284415766596794 + }, + "statuses": { + "200": 8703, + "201": 967, + "202": 486, + "503": 38 + } + }, + "successful_rps": 2.999141652669939, + "error_ratio": 0.7169811320754718, + "latency_success_ms": { + "0.5": 571.0092498920858, + "0.95": 1970.5530409701169, + "0.99": 1970.5530409701169 + }, + "statuses": { + "200": 11, + "201": 1, + "202": 3, + "503": 38 + } + }, + { + "name": "db_short_recovery_1", + "start": 1794816468.896548, + "end": 1794816528.899312, + "before": { + "requests": 10196, + "success": 10158, + "errors": 38, + "generator_dropped": 0, + "uploads_accepted": 486, + "latency_success_ms": { + "0.5": 8.442915976047516, + "0.95": 16.664416063576937, + "0.99": 29.284415766596794 + }, + "statuses": { + "200": 8705, + "201": 967, + "202": 486, + "503": 38 + } + }, + "after": { + "requests": 10825, + "success": 10787, + "errors": 38, + "generator_dropped": 0, + "uploads_accepted": 516, + "latency_success_ms": { + "0.5": 8.412291295826435, + "0.95": 16.663665883243084, + "0.99": 29.284415766596794 + }, + "statuses": { + "200": 9244, + "201": 1027, + "202": 516, + "503": 38 + } + }, + "successful_rps": 10.499526462454204, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.057749830186367, + "0.95": 16.250416170805693, + "0.99": 28.83725007995963 + }, + "statuses": { + "200": 540, + "201": 60, + "202": 30 + } + } + ], + "image_id": "sha256:5157155d7b9bee87926b5d1cde95140a9118fce6e2b0d621e5e59c19bcf6d304", + "completed": false, + "failure_type": "KeyboardInterrupt", + "finished": 1794816741.2585962, + "traffic": { + "requests": 13055, + "success": 13017, + "errors": 38, + "generator_dropped": 0, + "uploads_accepted": 622, + "latency_success_ms": { + "0.5": 8.275915868580341, + "0.95": 16.496040858328342, + "0.99": 28.75720802694559 + }, + "statuses": { + "200": 11155, + "201": 1240, + "202": 622, + "503": 38 + } + } +} diff --git a/evidence/aborted-counter-slo-run.txt b/evidence/aborted-counter-slo-run.txt new file mode 100644 index 0000000..8d21b30 --- /dev/null +++ b/evidence/aborted-counter-slo-run.txt @@ -0,0 +1,45 @@ +baseline 900 seconds +baseline 2 s 24 requests +baseline 66 s 698 requests +baseline 131 s 1378 requests +baseline 195 s 2049 requests +baseline 260 s 2728 requests +baseline 324 s 3404 requests +baseline 389 s 4083 requests +baseline 453 s 4758 requests +baseline 517 s 5434 requests +baseline 582 s 6116 requests +baseline 646 s 6789 requests +baseline 711 s 7469 requests +baseline 775 s 8140 requests +baseline 840 s 8819 requests +worker_short_1 5 seconds +worker_short_1 2 s 9479 requests +worker_short_recovery_1 60 seconds +worker_short_recovery_1 2 s 9532 requests +db_short_1 5 seconds +db_short_1 2 s 10144 requests +db_short_recovery_1 60 seconds +db_short_recovery_1 2 s 10219 requests +worker_long_1 240 seconds +worker_long_1 2 s 10849 requests +worker_long_1 66 s 11525 requests +worker_long_1 131 s 12207 requests +worker_long_1 194 s 12869 requests +Traceback (most recent call last): + File "/scripts/experiment-slo.py", line 147, in + if __name__=='__main__':main() + ^^^^^^ + File "/scripts/experiment-slo.py", line 119, in main + worker(a.long,f'worker_long_{i}');phase(f'worker_recovery_{i}',a.cooldown) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scripts/experiment-slo.py", line 99, in worker + try:phase(name,duration) + ^^^^^^^^^^^^^^^^^^^^ + File "/scripts/experiment-slo.py", line 92, in phase + save();time.sleep(min(5,max(0,at+seconds-time.time()))) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scripts/experiment-slo.py", line 67, in interrupted + def interrupted(*_):raise KeyboardInterrupt() + ^^^^^^^^^^^^^^^^^^^^^^^^^ +KeyboardInterrupt diff --git a/evidence/backend-checks.txt b/evidence/backend-checks.txt new file mode 100644 index 0000000..ef0f764 --- /dev/null +++ b/evidence/backend-checks.txt @@ -0,0 +1 @@ +Saved backend outage balance, queue, memory and business persistence evidence diff --git a/evidence/backend-outage.json b/evidence/backend-outage.json new file mode 100644 index 0000000..0a7fa92 --- /dev/null +++ b/evidence/backend-outage.json @@ -0,0 +1,1979 @@ +{ + "outage_seconds": 300.0064158439636, + "before": { + "otelcol_receiver_accepted_spans_total": [ + { + "metric": { + "__name__": "otelcol_receiver_accepted_spans_total", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/receiver/receiverhelper", + "receiver": "otlp", + "transport": "http" + }, + "value": [ + 1794823775.76, + "1299025" + ] + } + ], + "otelcol_exporter_sent_spans_total": [ + { + "metric": { + "__name__": "otelcol_exporter_sent_spans_total", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794823775.76, + "1298978" + ] + } + ], + "otelcol_exporter_send_failed_spans_total": [ + { + "metric": { + "__name__": "otelcol_exporter_send_failed_spans_total", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794823775.76, + "0" + ] + } + ], + "otelcol_exporter_enqueue_failed_spans_total": [], + "otelcol_receiver_refused_spans_total": [ + { + "metric": { + "__name__": "otelcol_receiver_refused_spans_total", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/receiver/receiverhelper", + "receiver": "otlp", + "transport": "http" + }, + "value": [ + 1794823775.76, + "0" + ] + } + ], + "otelcol_exporter_queue_size": [ + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "traces", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794823775.76, + "0" + ] + } + ], + "otelcol_exporter_queue_capacity": [ + { + "metric": { + "__name__": "otelcol_exporter_queue_capacity", + "data_type": "traces", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794823775.76, + "32" + ] + } + ] + }, + "after_drain": { + "otelcol_receiver_accepted_spans_total": [ + { + "metric": { + "__name__": "otelcol_receiver_accepted_spans_total", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/receiver/receiverhelper", + "receiver": "otlp", + "transport": "http" + }, + "value": [ + 1794824166.208, + "1350623" + ] + } + ], + "otelcol_exporter_sent_spans_total": [ + { + "metric": { + "__name__": "otelcol_exporter_sent_spans_total", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794824166.208, + "1312394" + ] + } + ], + "otelcol_exporter_send_failed_spans_total": [ + { + "metric": { + "__name__": "otelcol_exporter_send_failed_spans_total", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794824166.208, + "2524" + ] + } + ], + "otelcol_exporter_enqueue_failed_spans_total": [ + { + "metric": { + "__name__": "otelcol_exporter_enqueue_failed_spans_total", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794824166.208, + "35676" + ] + } + ], + "otelcol_receiver_refused_spans_total": [ + { + "metric": { + "__name__": "otelcol_receiver_refused_spans_total", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/receiver/receiverhelper", + "receiver": "otlp", + "transport": "http" + }, + "value": [ + 1794824166.208, + "0" + ] + } + ], + "otelcol_exporter_queue_size": [ + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "traces", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794824166.208, + "0" + ] + } + ], + "otelcol_exporter_queue_capacity": [ + { + "metric": { + "__name__": "otelcol_exporter_queue_capacity", + "data_type": "traces", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794824166.208, + "32" + ] + } + ] + }, + "counter_deltas": { + "otelcol_receiver_accepted_spans_total": 51598.0, + "otelcol_exporter_sent_spans_total": 13416.0, + "otelcol_exporter_send_failed_spans_total": 2524.0, + "otelcol_exporter_enqueue_failed_spans_total": 35676.0, + "otelcol_receiver_refused_spans_total": 0.0 + }, + "collector_accepted_minus_sent_estimate": 38182.0, + "estimate_note": "Background health/observer spans remain active; balance is approximate by a few in-flight batches. SDK-side losses are not included; send_failed counts attempts, not unique spans.", + "peak_exporter_queue": 32.0, + "queue_samples": [ + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "traces", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "values": [ + [ + 1794823775.76, + "0" + ], + [ + 1794823780.76, + "4" + ], + [ + 1794823785.76, + "9" + ], + [ + 1794823790.76, + "15" + ], + [ + 1794823795.76, + "20" + ], + [ + 1794823800.76, + "26" + ], + [ + 1794823805.76, + "32" + ], + [ + 1794823810.76, + "32" + ], + [ + 1794823815.76, + "32" + ], + [ + 1794823820.76, + "32" + ], + [ + 1794823825.76, + "32" + ], + [ + 1794823830.76, + "32" + ], + [ + 1794823835.76, + "31" + ], + [ + 1794823840.76, + "32" + ], + [ + 1794823845.76, + "32" + ], + [ + 1794823850.76, + "32" + ], + [ + 1794823855.76, + "32" + ], + [ + 1794823860.76, + "32" + ], + [ + 1794823865.76, + "32" + ], + [ + 1794823870.76, + "32" + ], + [ + 1794823875.76, + "32" + ], + [ + 1794823880.76, + "32" + ], + [ + 1794823885.76, + "32" + ], + [ + 1794823890.76, + "32" + ], + [ + 1794823895.76, + "32" + ], + [ + 1794823900.76, + "32" + ], + [ + 1794823905.76, + "32" + ], + [ + 1794823910.76, + "32" + ], + [ + 1794823915.76, + "32" + ], + [ + 1794823920.76, + "32" + ], + [ + 1794823925.76, + "32" + ], + [ + 1794823930.76, + "32" + ], + [ + 1794823935.76, + "32" + ], + [ + 1794823940.76, + "32" + ], + [ + 1794823945.76, + "31" + ], + [ + 1794823950.76, + "32" + ], + [ + 1794823955.76, + "32" + ], + [ + 1794823960.76, + "32" + ], + [ + 1794823965.76, + "32" + ], + [ + 1794823970.76, + "32" + ], + [ + 1794823975.76, + "32" + ], + [ + 1794823980.76, + "32" + ], + [ + 1794823985.76, + "32" + ], + [ + 1794823990.76, + "32" + ], + [ + 1794823995.76, + "32" + ], + [ + 1794824000.76, + "32" + ], + [ + 1794824005.76, + "32" + ], + [ + 1794824010.76, + "32" + ], + [ + 1794824015.76, + "32" + ], + [ + 1794824020.76, + "32" + ], + [ + 1794824025.76, + "32" + ], + [ + 1794824030.76, + "32" + ], + [ + 1794824035.76, + "32" + ], + [ + 1794824040.76, + "32" + ], + [ + 1794824045.76, + "32" + ], + [ + 1794824050.76, + "32" + ], + [ + 1794824055.76, + "32" + ], + [ + 1794824060.76, + "32" + ], + [ + 1794824065.76, + "32" + ], + [ + 1794824070.76, + "32" + ], + [ + 1794824075.76, + "32" + ], + [ + 1794824080.76, + "1" + ], + [ + 1794824085.76, + "0" + ], + [ + 1794824090.76, + "0" + ], + [ + 1794824095.76, + "0" + ], + [ + 1794824100.76, + "0" + ], + [ + 1794824105.76, + "0" + ], + [ + 1794824110.76, + "0" + ], + [ + 1794824115.76, + "0" + ], + [ + 1794824120.76, + "0" + ], + [ + 1794824125.76, + "0" + ], + [ + 1794824130.76, + "0" + ], + [ + 1794824135.76, + "0" + ], + [ + 1794824140.76, + "0" + ], + [ + 1794824145.76, + "0" + ], + [ + 1794824150.76, + "0" + ], + [ + 1794824155.76, + "0" + ], + [ + 1794824160.76, + "0" + ], + [ + 1794824165.76, + "0" + ] + ] + } + ], + "collector_memory_peak_bytes": 98398371.84, + "collector_memory_limit_bytes": 268435456, + "drop_records": [ + { + "rejected_items": 106, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 117, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 133, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 101, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 139, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 99, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 122, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 150, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 80, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 131, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 124, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 75, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 150, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 74, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 137, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 113, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 91, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 150, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 71, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 138, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 77, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 121, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 118, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 83, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 153, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 72, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 136, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 99, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 124, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 135, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 70, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 129, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 163, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 125, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 132, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 94, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 135, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 73, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 131, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 98, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 123, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 135, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 73, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 143, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 126, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 133, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 141, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 88, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 152, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 72, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 132, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 137, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 62, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 124, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 116, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 119, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 80, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 115, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 120, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 154, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 100, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 112, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 117, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 131, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 128, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 129, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 103, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 121, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 151, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 106, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 123, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 129, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 142, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 128, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 72, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 160, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 122, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 159, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 70, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 150, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 121, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 133, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 78, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 133, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 71, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 128, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 132, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 156, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 129, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 87, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 145, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 122, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 132, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 103, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 125, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 82, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 139, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 116, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 136, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 102, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 124, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 162, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 114, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 105, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 151, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 126, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 99, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 124, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 148, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 108, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 142, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 90, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 118, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 132, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 117, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 137, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 107, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 114, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 156, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 126, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 115, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 108, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 136, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 125, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 147, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 78, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 151, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 109, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 115, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 131, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 96, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 125, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 131, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 147, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 74, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 133, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 133, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 102, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 142, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 119, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 82, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 122, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 152, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 128, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 106, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 121, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 150, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 119, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 134, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 103, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 122, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 142, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 115, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 158, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 164, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 68, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 151, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 120, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 131, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 113, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 116, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 135, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 122, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 166, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 121, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 125, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 107, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 124, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 106, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 125, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 147, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 107, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 161, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 127, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 155, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 74, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 155, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 124, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 122, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 147, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 107, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 97, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 102, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 136, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 61, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 133, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 88, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 129, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 149, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 108, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 148, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 81, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 154, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 148, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 128, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 108, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 134, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 81, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 143, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 87, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 159, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 113, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 131, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 143, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 140, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 125, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 131, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 100, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 129, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 110, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 122, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 150, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 113, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 135, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 144, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 141, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 110, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 141, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 82, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 152, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 107, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 122, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 129, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 121, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 140, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 93, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 114, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 151, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 72, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 132, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 124, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 149, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 132, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 104, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 121, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 153, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 138, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 116, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 142, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 131, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 179, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 55, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 150, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 81, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 146, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 100, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 130, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 123, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 132, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 73, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 161, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 75, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 155, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "dropped_items": 126, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 133, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 85, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 149, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 80, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 159, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 122, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 128, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 74, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 131, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 113, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 163, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 103, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 156, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 73, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 122, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + }, + { + "rejected_items": 134, + "otelcol.component.id": "otlphttp/tempo", + "otelcol.signal": "traces" + } + ], + "drop_records_items": 2524, + "message_persistence": { + "confirmed": 6334, + "stored": 6334, + "confirmed_missing": 0 + }, + "attachment_states": { + "ready": 3529 + } +} diff --git a/evidence/broker-persistence.json b/evidence/broker-persistence.json new file mode 100644 index 0000000..23d72b3 --- /dev/null +++ b/evidence/broker-persistence.json @@ -0,0 +1,10 @@ +{ + "result": "PASS", + "attachment_id": "6e909ff4-fb98-496d-a0bf-adc4ac734463", + "broker_hostname": "broker", + "old_container_id": "4ad7e02f4e35c2a6907afafb3a4bcb48c13c4ddefb09322b46bba644984b3e04", + "new_container_id": "56a8639818cd4c10d3c6f521f03a72132c0535824f5bd815906cefc717b6893a", + "message_present_before_and_after": true, + "outbox_published_before_replacement": true, + "dispatcher_stopped_during_replacement": true +} diff --git a/evidence/build.txt b/evidence/build.txt new file mode 100644 index 0000000..c5df7d2 --- /dev/null +++ b/evidence/build.txt @@ -0,0 +1,3 @@ +cd service && go build ./... +cd service && go build -o ../bin/benchmark ./cmd/benchmark +cd service && go vet ./... diff --git a/evidence/collector-config.txt b/evidence/collector-config.txt new file mode 100644 index 0000000..db8438d --- /dev/null +++ b/evidence/collector-config.txt @@ -0,0 +1,2 @@ + Container mtusi2026lab6-otel-collector-run-63fc05410c49 Creating + Container mtusi2026lab6-otel-collector-run-63fc05410c49 Created diff --git a/evidence/contracts.txt b/evidence/contracts.txt new file mode 100644 index 0000000..ed9cd47 --- /dev/null +++ b/evidence/contracts.txt @@ -0,0 +1,4 @@ +.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/correlation-check.txt b/evidence/correlation-check.txt new file mode 100644 index 0000000..51747f5 --- /dev/null +++ b/evidence/correlation-check.txt @@ -0,0 +1,3 @@ +python3 scripts/prove-correlation.py +PASS: correlated upload, retry=False +PASS: correlated upload, retry=True diff --git a/evidence/correlation.json b/evidence/correlation.json new file mode 100644 index 0000000..bde4e4c --- /dev/null +++ b/evidence/correlation.json @@ -0,0 +1,1201 @@ +[ + { + "trace_id": "e76b6392d83e44d4b369e660595f194f", + "retry": false, + "attachment_id": "c5d72a10-0651-4880-8bbc-d226419ea5ad", + "job_id": "dad9eba4-6b23-4393-9aa9-c8a2032af231", + "attempt": 1, + "spans": [ + { + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "Hb2J6uNhYho=", + "parentSpanId": "xR222812SVU=", + "startTimeUnixNano": "1794906708003693548", + "endTimeUnixNano": "1794906708026600923", + "status": {}, + "service": "messenger-api" + }, + { + "name": "auth.VerifySession", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "iehEdZyPJ1c=", + "parentSpanId": "Hb2J6uNhYho=", + "startTimeUnixNano": "1794906708003707423", + "endTimeUnixNano": "1794906708004102381", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "vtS2f3Ljw9A=", + "parentSpanId": "iehEdZyPJ1c=", + "startTimeUnixNano": "1794906708003785381", + "endTimeUnixNano": "1794906708004091798", + "status": {}, + "service": "messenger-api" + }, + { + "name": "auth.validateStoredSession", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "BJ46zzBb/GM=", + "parentSpanId": "iehEdZyPJ1c=", + "startTimeUnixNano": "1794906708004100131", + "endTimeUnixNano": "1794906708004102214", + "status": {}, + "service": "messenger-api" + }, + { + "name": "attachment.upload", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "chTDiWJETLE=", + "parentSpanId": "Hb2J6uNhYho=", + "startTimeUnixNano": "1794906708004107214", + "endTimeUnixNano": "1794906708026537798", + "status": {}, + "service": "messenger-api" + }, + { + "name": "chat.GetForMember", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "08yquJwH0Nk=", + "parentSpanId": "chTDiWJETLE=", + "startTimeUnixNano": "1794906708004108006", + "endTimeUnixNano": "1794906708006667131", + "status": {}, + "service": "messenger-api" + }, + { + "name": "chat.cachedChat", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "2XCl4tVayCg=", + "parentSpanId": "08yquJwH0Nk=", + "startTimeUnixNano": "1794906708004108464", + "endTimeUnixNano": "1794906708004251214", + "status": {}, + "service": "messenger-api" + }, + { + "name": "chat.cachedChat", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "OBJOX7xJnIQ=", + "parentSpanId": "08yquJwH0Nk=", + "startTimeUnixNano": "1794906708004272173", + "endTimeUnixNano": "1794906708004305339", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "DjC5LyvGPuE=", + "parentSpanId": "08yquJwH0Nk=", + "startTimeUnixNano": "1794906708004497173", + "endTimeUnixNano": "1794906708005303756", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "aV7aSQIW3bY=", + "parentSpanId": "chTDiWJETLE=", + "startTimeUnixNano": "1794906708011929048", + "endTimeUnixNano": "1794906708012436131", + "status": {}, + "service": "messenger-api" + }, + { + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "zjINuShN3jI=", + "parentSpanId": "chTDiWJETLE=", + "startTimeUnixNano": "1794906708012450173", + "endTimeUnixNano": "1794906708015989882", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "IS545B1hfBE=", + "parentSpanId": "chTDiWJETLE=", + "startTimeUnixNano": "1794906708018411298", + "endTimeUnixNano": "1794906708018478548", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "8eWnB7p32cw=", + "parentSpanId": "chTDiWJETLE=", + "startTimeUnixNano": "1794906708018486673", + "endTimeUnixNano": "1794906708021062923", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "TByptUnuJfc=", + "parentSpanId": "chTDiWJETLE=", + "startTimeUnixNano": "1794906708021077173", + "endTimeUnixNano": "1794906708021305048", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "+Mm0VYTcdxU=", + "parentSpanId": "chTDiWJETLE=", + "startTimeUnixNano": "1794906708021311839", + "endTimeUnixNano": "1794906708022533839", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "p+8bKErWYLU=", + "parentSpanId": "chTDiWJETLE=", + "startTimeUnixNano": "1794906708022559048", + "endTimeUnixNano": "1794906708023554173", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "RLFarRn/OX8=", + "parentSpanId": "chTDiWJETLE=", + "startTimeUnixNano": "1794906708023560839", + "endTimeUnixNano": "1794906708026297089", + "status": {}, + "service": "messenger-api" + }, + { + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "8KFLQlDcTJs=", + "parentSpanId": "chTDiWJETLE=", + "startTimeUnixNano": "1794906708033902423", + "endTimeUnixNano": "1794906708038896339", + "status": {}, + "service": "messenger-dispatcher" + }, + { + "name": "attachments consume", + "kind": "SPAN_KIND_CONSUMER", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "QxfhdD7s/Mo=", + "parentSpanId": "8KFLQlDcTJs=", + "startTimeUnixNano": "1794906708038414631", + "endTimeUnixNano": "1794906708081548673", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "attachment.process", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "BIG+pLjHdhU=", + "parentSpanId": "QxfhdD7s/Mo=", + "startTimeUnixNano": "1794906708038462298", + "endTimeUnixNano": "1794906708081546132", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "YqFt0uO5hPI=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708038474423", + "endTimeUnixNano": "1794906708038542131", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "WKEukbPLH64=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708038550589", + "endTimeUnixNano": "1794906708038973589", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "tiL2ohbJyKU=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708038981798", + "endTimeUnixNano": "1794906708039087965", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "9CqlHYazFGc=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708039101464", + "endTimeUnixNano": "1794906708040116964", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "yll3KiX2apk=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708040128589", + "endTimeUnixNano": "1794906708040271464", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "8TiVHDtZ33o=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708040276964", + "endTimeUnixNano": "1794906708040718256", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "s3.get", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "2kI2fK1uGSo=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708040833173", + "endTimeUnixNano": "1794906708042608090", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "image.transform", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "k6SLW85TP5A=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708047079881", + "endTimeUnixNano": "1794906708071357173", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "lmJi7Yg6KH4=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708071382714", + "endTimeUnixNano": "1794906708074287464", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "x6cDQPlPxYs=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708074319256", + "endTimeUnixNano": "1794906708075923631", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "5ALU24pTe3w=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708075948089", + "endTimeUnixNano": "1794906708076988673", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "ZNYtUSa2fYc=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708077006964", + "endTimeUnixNano": "1794906708078164131", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "w73MVTI7NKg=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708078284756", + "endTimeUnixNano": "1794906708078380589", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "1QiJd7OEZT0=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708078389673", + "endTimeUnixNano": "1794906708078585298", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "/pXIQQ/r4a0=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708078590714", + "endTimeUnixNano": "1794906708078741631", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "Wo15cNx1yiQ=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708078754214", + "endTimeUnixNano": "1794906708080122631", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "NRZvxlqCYdg=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708080128548", + "endTimeUnixNano": "1794906708080220548", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "yeEcpD5dIIc=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708080226548", + "endTimeUnixNano": "1794906708080293631", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "ZYxG+dnNxoY=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708080294964", + "endTimeUnixNano": "1794906708080355298", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "EsaFEi11Tdk=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708080360714", + "endTimeUnixNano": "1794906708080456339", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "N6+OGRmnXPw=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708080473006", + "endTimeUnixNano": "1794906708080854756", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "1LYdyusYq04=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708080860589", + "endTimeUnixNano": "1794906708081207506", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "52tjktg+RNSzaeZgWV8ZTw==", + "spanId": "ReDkx41x+pU=", + "parentSpanId": "BIG+pLjHdhU=", + "startTimeUnixNano": "1794906708081212964", + "endTimeUnixNano": "1794906708081495964", + "status": {}, + "service": "messenger-worker" + } + ], + "logs": [ + { + "service": "messenger-worker", + "trace_id": "e76b6392d83e44d4b369e660595f194f", + "span_id": "4317e1743eecfcca", + "event": "job_received", + "timestamp": "1794906708038419673" + }, + { + "service": "messenger-api", + "trace_id": "e76b6392d83e44d4b369e660595f194f", + "span_id": "d3ccaab89c07d0d9", + "event": "chat_db_lookup", + "timestamp": "1794906708004310756" + }, + { + "service": "messenger-api", + "trace_id": "e76b6392d83e44d4b369e660595f194f", + "span_id": "1dbd89eae361621a", + "event": "http_request", + "timestamp": "1794906708026579173" + }, + { + "service": "messenger-worker", + "trace_id": "e76b6392d83e44d4b369e660595f194f", + "span_id": "0481bea4b8c77615", + "event": "job_ready", + "timestamp": "1794906708081518256" + } + ] + }, + { + "trace_id": "1558bc561fd149319f7b4044693ca2dc", + "retry": true, + "attachment_id": "7f352174-b7b8-4538-9f95-e8b7bcca6889", + "job_id": "6f4da2fa-52f9-4ed3-88dd-ecb6c3bc966f", + "attempt": 2, + "spans": [ + { + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "UBc1dmBi0+E=", + "parentSpanId": "yYIzESKwTsg=", + "startTimeUnixNano": "1794906711342167091", + "endTimeUnixNano": "1794906711356631216", + "status": {}, + "service": "messenger-api" + }, + { + "name": "auth.VerifySession", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "ZhftDAuaQps=", + "parentSpanId": "UBc1dmBi0+E=", + "startTimeUnixNano": "1794906711342187174", + "endTimeUnixNano": "1794906711342712799", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "3xaDraZ+RBg=", + "parentSpanId": "ZhftDAuaQps=", + "startTimeUnixNano": "1794906711342304632", + "endTimeUnixNano": "1794906711342696132", + "status": {}, + "service": "messenger-api" + }, + { + "name": "auth.validateStoredSession", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "h+49IZHcPBg=", + "parentSpanId": "ZhftDAuaQps=", + "startTimeUnixNano": "1794906711342708507", + "endTimeUnixNano": "1794906711342712132", + "status": {}, + "service": "messenger-api" + }, + { + "name": "attachment.upload", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "hQn0JoSazzs=", + "parentSpanId": "UBc1dmBi0+E=", + "startTimeUnixNano": "1794906711342721424", + "endTimeUnixNano": "1794906711356547465", + "status": {}, + "service": "messenger-api" + }, + { + "name": "chat.GetForMember", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "ncjx/lnEu/s=", + "parentSpanId": "hQn0JoSazzs=", + "startTimeUnixNano": "1794906711342722132", + "endTimeUnixNano": "1794906711343016674", + "status": {}, + "service": "messenger-api" + }, + { + "name": "chat.cachedChat", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "63BxefGr1P4=", + "parentSpanId": "ncjx/lnEu/s=", + "startTimeUnixNano": "1794906711342722591", + "endTimeUnixNano": "1794906711343004757", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "eZVl2YlNu0Q=", + "parentSpanId": "hQn0JoSazzs=", + "startTimeUnixNano": "1794906711347657341", + "endTimeUnixNano": "1794906711347863966", + "status": {}, + "service": "messenger-api" + }, + { + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "sSjkqLpVMVo=", + "parentSpanId": "hQn0JoSazzs=", + "startTimeUnixNano": "1794906711347881091", + "endTimeUnixNano": "1794906711351045924", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "QslQHcgChpY=", + "parentSpanId": "hQn0JoSazzs=", + "startTimeUnixNano": "1794906711351258174", + "endTimeUnixNano": "1794906711351384757", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "fIbYt9ejZtA=", + "parentSpanId": "hQn0JoSazzs=", + "startTimeUnixNano": "1794906711351412757", + "endTimeUnixNano": "1794906711352200799", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "NBZYxi/3j04=", + "parentSpanId": "hQn0JoSazzs=", + "startTimeUnixNano": "1794906711352216632", + "endTimeUnixNano": "1794906711352958049", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "R2y56mSSvdY=", + "parentSpanId": "hQn0JoSazzs=", + "startTimeUnixNano": "1794906711352967674", + "endTimeUnixNano": "1794906711354187216", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "/t40iwD2Ev8=", + "parentSpanId": "hQn0JoSazzs=", + "startTimeUnixNano": "1794906711354211382", + "endTimeUnixNano": "1794906711355049174", + "status": {}, + "service": "messenger-api" + }, + { + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "dzWlDAzvA8w=", + "parentSpanId": "hQn0JoSazzs=", + "startTimeUnixNano": "1794906711355063549", + "endTimeUnixNano": "1794906711356306507", + "status": {}, + "service": "messenger-api" + }, + { + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "DY1WbhXrv7A=", + "parentSpanId": "hQn0JoSazzs=", + "startTimeUnixNano": "1794906711408750466", + "endTimeUnixNano": "1794906711413462716", + "status": {}, + "service": "messenger-dispatcher" + }, + { + "name": "attachments consume", + "kind": "SPAN_KIND_CONSUMER", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "c8FvMbz5gy8=", + "parentSpanId": "DY1WbhXrv7A=", + "startTimeUnixNano": "1794906711932085966", + "endTimeUnixNano": "1794906711956087591", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "attachment.process", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "X0o9Wc/Rs8Q=", + "parentSpanId": "c8FvMbz5gy8=", + "startTimeUnixNano": "1794906711932187008", + "endTimeUnixNano": "1794906711956067674", + "status": { + "message": "retry", + "code": "STATUS_CODE_ERROR" + }, + "service": "messenger-worker" + }, + { + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "vhLnHG9srzQ=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711935604341", + "endTimeUnixNano": "1794906711935766549", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "aR8mAS077mw=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711935782966", + "endTimeUnixNano": "1794906711936919507", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "w64cc3ESC2U=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711936942049", + "endTimeUnixNano": "1794906711937660466", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "4v210/ynCXw=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711937715674", + "endTimeUnixNano": "1794906711938185091", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "EqqM/ICO+bE=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711938197091", + "endTimeUnixNano": "1794906711938648008", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "mEKqzcY6Lkg=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711938662424", + "endTimeUnixNano": "1794906711939082424", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "s3.get", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "2ikDTaXG9kk=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711940846383", + "endTimeUnixNano": "1794906711944086008", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "UKcwVMmJh1s=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711953429383", + "endTimeUnixNano": "1794906711953601842", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "octeCN/ZCtE=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711953614258", + "endTimeUnixNano": "1794906711953855550", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "6rAEl6YrYJg=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711953862008", + "endTimeUnixNano": "1794906711954033383", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "3M1KmuUMY3A=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711954083549", + "endTimeUnixNano": "1794906711955013132", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "k2SoJFx+DtI=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711955030716", + "endTimeUnixNano": "1794906711955194633", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "wdw/+ccvvfQ=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711955206299", + "endTimeUnixNano": "1794906711955463507", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "HCHY+Fvknpk=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906711955471383", + "endTimeUnixNano": "1794906711956012299", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "attachments consume", + "kind": "SPAN_KIND_CONSUMER", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "vSX85REHyyQ=", + "parentSpanId": "RbAWSZFNM4Y=", + "startTimeUnixNano": "1794906712550743841", + "endTimeUnixNano": "1794906712606108966", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "attachment.process", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "L+dZao1aUKY=", + "parentSpanId": "vSX85REHyyQ=", + "startTimeUnixNano": "1794906712550843716", + "endTimeUnixNano": "1794906712606100424", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "S/U9O5Ty374=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712550865883", + "endTimeUnixNano": "1794906712551176341", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "vJaaXruAhho=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712551202508", + "endTimeUnixNano": "1794906712551799342", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "q6lTQ9xlQqo=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712551828550", + "endTimeUnixNano": "1794906712552256925", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "b2CSHAauvrk=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712552295591", + "endTimeUnixNano": "1794906712552637091", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "QjODloaV7nE=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712552668091", + "endTimeUnixNano": "1794906712553154258", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "YDzbpqilLWk=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712553170050", + "endTimeUnixNano": "1794906712553516675", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "s3.get", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "9HvbvWCEGWU=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712553865216", + "endTimeUnixNano": "1794906712556029632", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "image.transform", + "kind": "SPAN_KIND_INTERNAL", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "Iw74rAcRaNc=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712561931841", + "endTimeUnixNano": "1794906712595050091", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "9WndyE7St6I=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712595075883", + "endTimeUnixNano": "1794906712598248174", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "YcLYgrzyr1E=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712598407591", + "endTimeUnixNano": "1794906712599285508", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "BV2t0PmKAIM=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712599306175", + "endTimeUnixNano": "1794906712600027175", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "ZVsp5NqoL40=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712600048675", + "endTimeUnixNano": "1794906712600883550", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "aKjXCKUmSfM=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712601084883", + "endTimeUnixNano": "1794906712601165133", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "0G917ToOmmg=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712601173716", + "endTimeUnixNano": "1794906712601357341", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "iC2AR8ud8GY=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712601363550", + "endTimeUnixNano": "1794906712601473175", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "6WfGtZdW4LE=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712601485758", + "endTimeUnixNano": "1794906712603090299", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "AS9UyZzkiis=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712603096508", + "endTimeUnixNano": "1794906712603205258", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "fbvHgrGX15E=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712603210550", + "endTimeUnixNano": "1794906712603340050", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "r012guOEVMQ=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712603345925", + "endTimeUnixNano": "1794906712603463175", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "4y68pmRYqPs=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712603469841", + "endTimeUnixNano": "1794906712603582883", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "6o6+wYHgguk=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712603611341", + "endTimeUnixNano": "1794906712604873758", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "SWRuSeR5Bd4=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712604879175", + "endTimeUnixNano": "1794906712605123341", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "ZpIwxVKHljE=", + "parentSpanId": "L+dZao1aUKY=", + "startTimeUnixNano": "1794906712605128800", + "endTimeUnixNano": "1794906712605966425", + "status": {}, + "service": "messenger-worker" + }, + { + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "traceId": "FVi8Vh/RSTGfe0BEaTyi3A==", + "spanId": "RbAWSZFNM4Y=", + "parentSpanId": "X0o9Wc/Rs8Q=", + "startTimeUnixNano": "1794906712539160508", + "endTimeUnixNano": "1794906712552213383", + "status": {}, + "service": "messenger-dispatcher" + } + ], + "logs": [ + { + "service": "messenger-worker", + "trace_id": "1558bc561fd149319f7b4044693ca2dc", + "span_id": "73c16f31bcf9832f", + "event": "job_received", + "timestamp": "1794906711932098799" + }, + { + "service": "messenger-worker", + "trace_id": "1558bc561fd149319f7b4044693ca2dc", + "span_id": "bd25fce51107cb24", + "event": "job_received", + "timestamp": "1794906712550754466" + }, + { + "service": "messenger-api", + "trace_id": "1558bc561fd149319f7b4044693ca2dc", + "span_id": "501735766062d3e1", + "event": "http_request", + "timestamp": "1794906711356601757" + }, + { + "service": "messenger-worker", + "trace_id": "1558bc561fd149319f7b4044693ca2dc", + "span_id": "2fe7596a8d5a50a6", + "event": "job_ready", + "timestamp": "1794906712606071966" + }, + { + "service": "messenger-worker", + "trace_id": "1558bc561fd149319f7b4044693ca2dc", + "span_id": "5f4a3d59cfd1b3c4", + "event": "job_retry", + "timestamp": "1794906711956030216" + } + ] + } +] diff --git a/evidence/delivery-checks.json b/evidence/delivery-checks.json new file mode 100644 index 0000000..3ac884e --- /dev/null +++ b/evidence/delivery-checks.json @@ -0,0 +1,10 @@ +{ + "verified_at_utc": "2026-11-17T09:13:49.849327+00:00", + "checked_files": 259, + "generated_secret_values_absent": true, + "secret_files_ignored": true, + "absolute_repository_paths_removed_from_text_evidence": true, + "current_smoke_passed": true, + "current_correlation_and_privacy_checks_passed": true, + "dashboard_pool_worker_backlog_exporter_have_separate_queries": true +} diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 0000000..50b321d --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,194 @@ +{ + "timestamp": 1794906731.3549702, + "revision": "8b381c115209cf7ac143df4826c468bf38128d71", + "working_tree_dirty": true, + "source_sha256": "efaad10cbbe055be8252463aaab6258fcc6b10b2e20211b4dcd56195bcecc991", + "platform": "macOS-26.5.1-arm64-arm-64bit", + "go": "go version go1.27.0 darwin/arm64", + "docker": { + "server_version": "29.7.2", + "cpus": 14, + "memory": 8320954368, + "architecture": "aarch64" + }, + "containers": [ + { + "service": "api", + "image": "mtusi-messenger:lab6", + "image_id": "sha256:c7021c3fed2700b5234fedad71dd0478acb126a697aa25ec5ac1a2c99020962b", + "started_at": "2026-11-16T10:20:53.477815219Z", + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "mounts": [] + }, + { + "service": "api", + "image": "mtusi-messenger:lab6", + "image_id": "sha256:c7021c3fed2700b5234fedad71dd0478acb126a697aa25ec5ac1a2c99020962b", + "started_at": "2026-11-16T10:20:53.332318052Z", + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "mounts": [] + }, + { + "service": "broker", + "image": "rabbitmq:4.2.0-alpine@sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524", + "image_id": "sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524", + "started_at": "2026-11-16T10:17:58.357427221Z", + "memory_limit": 536870912, + "nano_cpus": 500000000, + "mounts": [ + { + "type": "volume", + "name": "mtusi2026lab6_broker_data", + "destination": "/var/lib/rabbitmq" + } + ] + }, + { + "service": "dispatcher", + "image": "mtusi-messenger:lab6", + "image_id": "sha256:c7021c3fed2700b5234fedad71dd0478acb126a697aa25ec5ac1a2c99020962b", + "started_at": "2026-11-16T10:20:53.325688052Z", + "memory_limit": 134217728, + "nano_cpus": 250000000, + "mounts": [] + }, + { + "service": "grafana", + "image": "grafana/grafana:13.3.1", + "image_id": "sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf", + "started_at": "2026-11-16T10:16:16.370235216Z", + "memory_limit": 402653184, + "nano_cpus": 500000000, + "mounts": [ + { + "type": "volume", + "name": "mtusi2026lab6_grafana_data", + "destination": "/var/lib/grafana" + } + ] + }, + { + "service": "lb", + "image": "haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0", + "image_id": "sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0", + "started_at": "2026-11-16T10:17:15.316601507Z", + "memory_limit": 134217728, + "nano_cpus": 500000000, + "mounts": [] + }, + { + "service": "loki", + "image": "grafana/loki:3.7.10", + "image_id": "sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7", + "started_at": "2026-11-16T10:16:18.125747508Z", + "memory_limit": 402653184, + "nano_cpus": 500000000, + "mounts": [ + { + "type": "volume", + "name": "mtusi2026lab6_loki_data", + "destination": "/loki" + } + ] + }, + { + "service": "otel-collector", + "image": "otel/opentelemetry-collector-contrib:0.140.0", + "image_id": "sha256:ffc818ee108d0b934fd14207fd87ff247c9b64f344ed349d0b66c166c18a2312", + "started_at": "2026-11-16T10:16:17.9785623Z", + "memory_limit": 268435456, + "nano_cpus": 500000000, + "mounts": [] + }, + { + "service": "postgres", + "image": "postgres:17.11-alpine3.23@sha256:05aa77262db9277fde6374bb2f901c9604b8a9373029e1a7993440c6aae48736", + "image_id": "sha256:05aa77262db9277fde6374bb2f901c9604b8a9373029e1a7993440c6aae48736", + "started_at": "2026-11-16T10:17:57.426626804Z", + "memory_limit": 1073741824, + "nano_cpus": 2000000000, + "mounts": [ + { + "type": "volume", + "name": "mtusi2026lab6_postgres_data", + "destination": "/var/lib/postgresql/data" + } + ] + }, + { + "service": "prometheus", + "image": "prom/prometheus:v3.16.0", + "image_id": "sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06", + "started_at": "2026-11-16T10:16:15.930341549Z", + "memory_limit": 402653184, + "nano_cpus": 500000000, + "mounts": [ + { + "type": "volume", + "name": "mtusi2026lab6_prometheus_data", + "destination": "/prometheus" + } + ] + }, + { + "service": "redis", + "image": "redis:8.2.11-alpine3.22@sha256:489a769e0a54dc245896132a15c72d2bc7296719954da99398f43a3877e5f8eb", + "image_id": "sha256:489a769e0a54dc245896132a15c72d2bc7296719954da99398f43a3877e5f8eb", + "started_at": "2026-11-16T10:17:57.309233929Z", + "memory_limit": 134217728, + "nano_cpus": 250000000, + "mounts": [ + { + "type": "volume", + "name": "mtusi2026lab6_redis_data", + "destination": "/data" + } + ] + }, + { + "service": "s3", + "image": "chrislusf/seaweedfs:4.52@sha256:57877e32bb90073b37dc109b438ba66e84fd5ed9ced5f5c04f56444ef243ce59", + "image_id": "sha256:57877e32bb90073b37dc109b438ba66e84fd5ed9ced5f5c04f56444ef243ce59", + "started_at": "2026-11-16T10:18:22.319458969Z", + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "mounts": [ + { + "type": "volume", + "name": "mtusi2026lab6_s3_data", + "destination": "/data" + } + ] + }, + { + "service": "tempo", + "image": "grafana/tempo:2.9.0", + "image_id": "sha256:65a5789759435f1ef696f1953258b9bbdb18eb571d5ce711ff812d2e128288a4", + "started_at": "2026-11-17T09:01:50.420850382Z", + "memory_limit": 402653184, + "nano_cpus": 500000000, + "mounts": [ + { + "type": "volume", + "name": "mtusi2026lab6_tempo_data", + "destination": "/var/tempo" + } + ] + }, + { + "service": "worker", + "image": "mtusi-messenger:lab6", + "image_id": "sha256:c7021c3fed2700b5234fedad71dd0478acb126a697aa25ec5ac1a2c99020962b", + "started_at": "2026-11-17T09:11:55.452974759Z", + "memory_limit": 1073741824, + "nano_cpus": 1000000000, + "mounts": [] + } + ], + "host": { + "cpu": "Apple M4 Pro", + "memory": "25769803776" + } +} diff --git a/evidence/failover-action.jsonl b/evidence/failover-action.jsonl new file mode 100644 index 0000000..db7dbdd --- /dev/null +++ b/evidence/failover-action.jsonl @@ -0,0 +1 @@ +{"at": 1794824242.3058822, "container": "747d3207a8824277939ee3d3727b9528eae03ae9f38e9d7a9d8c7c660b0e5cf8", "instance_id": "78f4a480-202b-41b7-a1a1-ab499d7be4c6", "action": "docker stop", "project": "mtusi2026lab6"} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..d506c92 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18086" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 1, "first_recovery_seconds": 1.148} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/final-checks.txt b/evidence/final-checks.txt new file mode 100644 index 0000000..1ef6875 --- /dev/null +++ b/evidence/final-checks.txt @@ -0,0 +1,94 @@ +python3 tests/smoke.py --base-url "http://localhost:18086" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +cd service && TEST_BASE_URL="http://localhost:18086" go test -race -count=1 -v ./... +? gitea.privcloud.tech/mtusi2026/cmd/app [no test files] +=== RUN TestLostPostResponseIsNotRetried +--- PASS: TestLostPostResponseIsNotRetried (0.00s) +PASS +ok gitea.privcloud.tech/mtusi2026/cmd/benchmark 1.439s +? gitea.privcloud.tech/mtusi2026/internal/adapter/broker [no test files] +? gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http [no test files] +? gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside [no test files] +=== RUN TestIdempotencyTransactionAndExpiry + idempotency_test.go:27: run make test-storage for an isolated PostgreSQL database +--- SKIP: TestIdempotencyTransactionAndExpiry (0.00s) +PASS +ok gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage 1.715s +? gitea.privcloud.tech/mtusi2026/internal/config [no test files] +? gitea.privcloud.tech/mtusi2026/internal/dto [no test files] +? gitea.privcloud.tech/mtusi2026/internal/gen/contracts [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/apperr [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/cache [no test files] +=== RUN TestSessionFingerprints +--- PASS: TestSessionFingerprints (0.03s) +PASS +ok gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting 1.341s +? gitea.privcloud.tech/mtusi2026/internal/pkg/fault [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/migrator [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers [no test files] +=== RUN TestVerifiedSessionIsRequestScoped +2026/11/16 13:21:03 ERROR http_error error_code=CodeUnauthorized status=401 request_id=a407e1e0-b4ee-4006-b34f-8256cb26951b +--- PASS: TestVerifiedSessionIsRequestScoped (0.00s) +=== RUN TestMissingTokenDoesNotCallSessionStore +2026/11/16 13:21:03 ERROR http_error error_code=CodeUnauthorized status=401 request_id=12e5acbe-dc1b-4b12-ab1a-d7bf23c5b0c2 +--- PASS: TestMissingTokenDoesNotCallSessionStore (0.00s) +PASS +ok gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware 1.992s +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/authprovider [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/role [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/validator [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/attachment [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/auth [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/chat [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/observation [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/outbox [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/user [no test files] +? gitea.privcloud.tech/mtusi2026/migrations [no test files] +=== RUN TestInvalidJSON +=== RUN TestInvalidJSON/malformed +=== RUN TestInvalidJSON/wrong_type +=== RUN TestInvalidJSON/unknown_field +--- PASS: TestInvalidJSON (0.01s) + --- PASS: TestInvalidJSON/malformed (0.01s) + --- PASS: TestInvalidJSON/wrong_type (0.00s) + --- PASS: TestInvalidJSON/unknown_field (0.00s) +=== RUN TestForeignChatIsHidden +--- PASS: TestForeignChatIsHidden (0.57s) +=== RUN TestEmptyMessage +--- PASS: TestEmptyMessage (0.19s) +=== RUN TestDuplicateUsername +--- PASS: TestDuplicateUsername (0.22s) +=== RUN TestMessagePagination +--- PASS: TestMessagePagination (0.20s) +=== RUN TestAttachmentContractAndErrors +=== RUN TestAttachmentContractAndErrors/fake +=== RUN TestAttachmentContractAndErrors/mime_mismatch +=== RUN TestAttachmentContractAndErrors/wrong_type +=== RUN TestAttachmentContractAndErrors/truncated +--- PASS: TestAttachmentContractAndErrors (1.05s) + --- PASS: TestAttachmentContractAndErrors/fake (0.00s) + --- PASS: TestAttachmentContractAndErrors/mime_mismatch (0.00s) + --- PASS: TestAttachmentContractAndErrors/wrong_type (0.00s) + --- PASS: TestAttachmentContractAndErrors/truncated (0.00s) +=== RUN TestConcurrentAttachmentMessage +--- PASS: TestConcurrentAttachmentMessage (0.20s) +=== RUN TestConcurrentMessages + concurrency_test.go:160: 10 clients sent 100 messages; history contains 100 unique messages across 6 pages +--- PASS: TestConcurrentMessages (1.46s) +=== RUN TestIdempotencyContract +--- PASS: TestIdempotencyContract (0.53s) +=== RUN TestConcurrentIdempotencyAcrossReplicas + idempotency_test.go:103: 32 requests across 2 replicas created exactly one message +--- PASS: TestConcurrentIdempotencyAcrossReplicas (0.22s) +PASS +ok gitea.privcloud.tech/mtusi2026/tests 6.932s diff --git a/evidence/first-spike-check.txt b/evidence/first-spike-check.txt new file mode 100644 index 0000000..971121f --- /dev/null +++ b/evidence/first-spike-check.txt @@ -0,0 +1 @@ +PASS: first 5xx spike appears in Prometheus increase: 31.11 diff --git a/evidence/first-spike.json b/evidence/first-spike.json new file mode 100644 index 0000000..0ebc760 --- /dev/null +++ b/evidence/first-spike.json @@ -0,0 +1,349 @@ +{ + "at": 1794817011.404624, + "before": [ + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794816991.111, + "15" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794816991.111, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "de655f64-de54-45b8-923b-ace3dde6c38e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "de655f64-de54-45b8-923b-ace3dde6c38e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794816991.111, + "12" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "de655f64-de54-45b8-923b-ace3dde6c38e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "de655f64-de54-45b8-923b-ace3dde6c38e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794816991.111, + "2" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "44064e41-06b3-4298-850f-67948dc8dd4c", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "44064e41-06b3-4298-850f-67948dc8dd4c", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794816991.111, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "44064e41-06b3-4298-850f-67948dc8dd4c", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "44064e41-06b3-4298-850f-67948dc8dd4c", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794816991.111, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "e57133fe-8e06-48a4-8984-b23d3e6bda08", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "e57133fe-8e06-48a4-8984-b23d3e6bda08", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794816991.111, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "e57133fe-8e06-48a4-8984-b23d3e6bda08", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "e57133fe-8e06-48a4-8984-b23d3e6bda08", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794816991.111, + "0" + ] + } + ], + "after": [ + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794817011.408, + "15" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "715b9cfb-8dd2-451b-89a4-5e2cbc4b4332", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794817011.408, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "de655f64-de54-45b8-923b-ace3dde6c38e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "de655f64-de54-45b8-923b-ace3dde6c38e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794817011.408, + "12" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "de655f64-de54-45b8-923b-ace3dde6c38e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "de655f64-de54-45b8-923b-ace3dde6c38e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794817011.408, + "2" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "44064e41-06b3-4298-850f-67948dc8dd4c", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "44064e41-06b3-4298-850f-67948dc8dd4c", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794817011.408, + "15" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "44064e41-06b3-4298-850f-67948dc8dd4c", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "44064e41-06b3-4298-850f-67948dc8dd4c", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794817011.408, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "e57133fe-8e06-48a4-8984-b23d3e6bda08", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "e57133fe-8e06-48a4-8984-b23d3e6bda08", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794817011.408, + "15" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "e57133fe-8e06-48a4-8984-b23d3e6bda08", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "e57133fe-8e06-48a4-8984-b23d3e6bda08", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794817011.408, + "0" + ] + } + ], + "increase_1m": 31.11488882020327, + "statuses": { + "200": 20, + "503": 30 + }, + "passed": true +} diff --git a/evidence/go-checks.txt b/evidence/go-checks.txt new file mode 100644 index 0000000..fdebec0 --- /dev/null +++ b/evidence/go-checks.txt @@ -0,0 +1,92 @@ +cd service && TEST_BASE_URL="http://localhost:18086" go test -race -count=1 -v ./... +? gitea.privcloud.tech/mtusi2026/cmd/app [no test files] +=== RUN TestLostPostResponseIsNotRetried +--- PASS: TestLostPostResponseIsNotRetried (0.00s) +PASS +ok gitea.privcloud.tech/mtusi2026/cmd/benchmark 1.364s +? gitea.privcloud.tech/mtusi2026/internal/adapter/broker [no test files] +? gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http [no test files] +? gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside [no test files] +=== RUN TestIdempotencyTransactionAndExpiry + idempotency_test.go:27: run make test-storage for an isolated PostgreSQL database +--- SKIP: TestIdempotencyTransactionAndExpiry (0.00s) +PASS +ok gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage 1.350s +? gitea.privcloud.tech/mtusi2026/internal/config [no test files] +? gitea.privcloud.tech/mtusi2026/internal/dto [no test files] +? gitea.privcloud.tech/mtusi2026/internal/gen/contracts [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/apperr [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/cache [no test files] +=== RUN TestSessionFingerprints +--- PASS: TestSessionFingerprints (0.02s) +PASS +ok gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting 1.493s +? gitea.privcloud.tech/mtusi2026/internal/pkg/fault [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/migrator [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers [no test files] +=== RUN TestVerifiedSessionIsRequestScoped +2026/11/16 10:13:58 ERROR response error="revoked session" status=401 requestId=5fc00aba-9637-4083-a41e-e825350baf99 +--- PASS: TestVerifiedSessionIsRequestScoped (0.00s) +=== RUN TestMissingTokenDoesNotCallSessionStore +2026/11/16 10:13:58 ERROR response error="missing or invalid bearer token" status=401 requestId=4fb2abe7-293f-4814-96dd-be8dc4a1b768 +--- PASS: TestMissingTokenDoesNotCallSessionStore (0.00s) +PASS +ok gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware 1.655s +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/authprovider [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/role [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/validator [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/attachment [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/auth [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/chat [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/observation [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/outbox [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/user [no test files] +? gitea.privcloud.tech/mtusi2026/migrations [no test files] +=== RUN TestInvalidJSON +=== RUN TestInvalidJSON/malformed +=== RUN TestInvalidJSON/wrong_type +=== RUN TestInvalidJSON/unknown_field +--- PASS: TestInvalidJSON (0.02s) + --- PASS: TestInvalidJSON/malformed (0.01s) + --- PASS: TestInvalidJSON/wrong_type (0.00s) + --- PASS: TestInvalidJSON/unknown_field (0.00s) +=== RUN TestForeignChatIsHidden +--- PASS: TestForeignChatIsHidden (0.61s) +=== RUN TestEmptyMessage +--- PASS: TestEmptyMessage (0.20s) +=== RUN TestDuplicateUsername +--- PASS: TestDuplicateUsername (0.29s) +=== RUN TestMessagePagination +--- PASS: TestMessagePagination (0.22s) +=== RUN TestAttachmentContractAndErrors +=== RUN TestAttachmentContractAndErrors/fake +=== RUN TestAttachmentContractAndErrors/mime_mismatch +=== RUN TestAttachmentContractAndErrors/wrong_type +=== RUN TestAttachmentContractAndErrors/truncated +--- PASS: TestAttachmentContractAndErrors (13.12s) + --- PASS: TestAttachmentContractAndErrors/fake (0.00s) + --- PASS: TestAttachmentContractAndErrors/mime_mismatch (0.00s) + --- PASS: TestAttachmentContractAndErrors/wrong_type (0.00s) + --- PASS: TestAttachmentContractAndErrors/truncated (0.01s) +=== RUN TestConcurrentAttachmentMessage +--- PASS: TestConcurrentAttachmentMessage (0.23s) +=== RUN TestConcurrentMessages + concurrency_test.go:160: 10 clients sent 100 messages; history contains 100 unique messages across 6 pages +--- PASS: TestConcurrentMessages (2.09s) +=== RUN TestIdempotencyContract +--- PASS: TestIdempotencyContract (0.56s) +=== RUN TestConcurrentIdempotencyAcrossReplicas + idempotency_test.go:103: 32 requests across 2 replicas created exactly one message +--- PASS: TestConcurrentIdempotencyAcrossReplicas (0.21s) +PASS +ok gitea.privcloud.tech/mtusi2026/tests 19.538s diff --git a/evidence/image-provenance.json b/evidence/image-provenance.json new file mode 100644 index 0000000..1e5f9cd --- /dev/null +++ b/evidence/image-provenance.json @@ -0,0 +1,47 @@ +{ + "research_tag_descriptor": "sha256:d661ef49e464a482dcae899acf1513fa1cf57fae01664445e2dbfb323e65c28a", + "pipeline_tag_descriptor": "sha256:c7021c3fed2700b5234fedad71dd0478acb126a697aa25ec5ac1a2c99020962b", + "runtime_platform_manifest": "sha256:cdbc329b56054f2a3effd832e70dcc50e8eff9b8b601515e2cf70f9de1e6d763", + "runtime_config": "sha256:90726e8699e10a67adece45ae424631076793a340b603ed12cadae16de4d6a45", + "runtime_manifest_present_in_research_build_log": true, + "compose_image_labels": { + "com.docker.compose.project": "mtusi2026lab6", + "com.docker.compose.service": "dispatcher", + "com.docker.compose.version": "5.3.1" + }, + "rootfs_layers": [ + "sha256:18932007cc273c0541920263e57c02d40013c4730d7db35437a4ffcc31422eda", + "sha256:3e2a611abc7aa98be9c8ce72079d284cc151fd38da2bcf878ba671bf756e2e7f", + "sha256:276547fba5cc9516e768b719cdff99136849d9059b464c0d7bb7322d26d4e9a6", + "sha256:e0fa85f3fe36e74b66b4b4dce01473ebabd7b983b3aa57800581c3901774b4a5" + ], + "messenger_binary_sha256": "33c6b59dd87a04c05cf8ff8ad0628fb302c361d32ef125204c4225c1f8337d62", + "note": "Compose builds the same Dockerfile for multiple roles and adds different service labels; OCI index/attestation digests differ. The runtime dispatcher platform manifest matches the research build export.", + "research_build_exports": { + "api": { + "platform_manifest": "sha256:fb582ac957d339c0e15c75ffdd5a469f037565dce4e04ff3eae9593e1a00ae59", + "config": "sha256:f0c85496a5195527417b80b23ac4764efa4c322ae8b655ffba02c14676428452", + "index": "sha256:daa0107d0ba7c7b03e5defcb1e0af9a768384eb3b56e3cb7df597287b06fc32f" + }, + "dispatcher": { + "platform_manifest": "sha256:cdbc329b56054f2a3effd832e70dcc50e8eff9b8b601515e2cf70f9de1e6d763", + "config": "sha256:90726e8699e10a67adece45ae424631076793a340b603ed12cadae16de4d6a45", + "index": "sha256:bd4d9ad2bf79cc347d125dbda02a4ddafc804196f6f9f0e2f1b68cdb9963e0a5" + }, + "worker": { + "platform_manifest": "sha256:6f8740eb7bdcd57abb07582d25c02b6b509f184c92ddc2a93ed1f21be1791f34", + "config": "sha256:e1220128a009c3c3031d20b771d5685ae8a51a1fd36ac049e6a9a62c351ea1e3", + "index": "sha256:9c8899b0323070c8d73943d0b782ae161ed42bf312ce3759bac0e65b737cad1c" + }, + "migrate": { + "platform_manifest": "sha256:dfcdf8937da1d219f4383c6f1f1057bc38713c17102e136321d371a2c5a888b9", + "config": "sha256:8afe86149f38a380ee2e3bbdd2cd8ec9c9437d3a401218191462a7708530fb2c", + "index": "sha256:00b2fe15bc327b038deeeb471cad000ad01e4c32c6f73039e3b5f57ab46d5963" + }, + "s3-init": { + "platform_manifest": "sha256:0374cc3c70e64749b51918452b598aad596a8a83896196f6a5b66eb954fc644e", + "config": "sha256:99c7245ce4f99b2f48429a11d1b80fb62604400d5e6feb3c3972219901076a6d", + "index": "sha256:d661ef49e464a482dcae899acf1513fa1cf57fae01664445e2dbfb323e65c28a" + } + } +} diff --git a/evidence/images.json b/evidence/images.json new file mode 100644 index 0000000..717ac91 --- /dev/null +++ b/evidence/images.json @@ -0,0 +1,53 @@ +{ + "results": [ + { + "case": "small", + "result": "PASS", + "thumbnail": [ + 16, + 8 + ] + }, + { + "case": "floor", + "result": "PASS", + "thumbnail": [ + 256, + 76 + ] + }, + { + "case": "alpha", + "result": "PASS", + "thumbnail": [ + 40, + 20 + ] + }, + { + "case": "orientation", + "result": "PASS", + "thumbnail": [ + 128, + 256 + ] + }, + { + "case": "thin", + "result": "PASS", + "thumbnail": [ + 1, + 256 + ] + }, + { + "case": "center_crop", + "result": "PASS", + "thumbnail": [ + 256, + 85 + ] + } + ], + "negative_cases": 4 +} diff --git a/evidence/incident-signals.json b/evidence/incident-signals.json new file mode 100644 index 0000000..28f9382 --- /dev/null +++ b/evidence/incident-signals.json @@ -0,0 +1,1682 @@ +{ + "evidence": { + "worker_long_1": { + "captured": 1794818184.4672298, + "observed": { + "at": 1794818182.217382, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "3.5714285714285715e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "5.9309107e+01" + } + ], + "pending": 33.0, + "db_lock_waiting": 0.0, + "queue_age": 64.524647, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + "worker_received_events": 0, + "accepted_upload_events": 35, + "diagnosis": "Queue age and ready burn increase while API accepts uploads and no worker consumption is observed" + }, + "worker_recovery_1": { + "captured": 1794818374.986302, + "trace_id": "87ebd5bd431e611a386af21794ac2f47", + "spans": [ + { + "service": "messenger-api", + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "span_id": "tz0NsiHCttw=", + "parent_span_id": null, + "duration_ms": 18.956916, + "status": {} + }, + { + "service": "messenger-api", + "name": "auth.VerifySession", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "gELX57+vTkk=", + "parent_span_id": "tz0NsiHCttw=", + "duration_ms": 0.638041, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "D+jkhtupYPg=", + "parent_span_id": "gELX57+vTkk=", + "duration_ms": 0.335334, + "status": {} + }, + { + "service": "messenger-api", + "name": "auth.validateStoredSession", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "p5gFc8iJZHQ=", + "parent_span_id": "gELX57+vTkk=", + "duration_ms": 0.003334, + "status": {} + }, + { + "service": "messenger-api", + "name": "attachment.upload", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "jB0/2W4MDhw=", + "parent_span_id": "tz0NsiHCttw=", + "duration_ms": 18.209625, + "status": {} + }, + { + "service": "messenger-api", + "name": "chat.GetForMember", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "nGwUSRk5LnA=", + "parent_span_id": "jB0/2W4MDhw=", + "duration_ms": 0.210417, + "status": {} + }, + { + "service": "messenger-api", + "name": "chat.cachedChat", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "j3dUIQf4eWA=", + "parent_span_id": "nGwUSRk5LnA=", + "duration_ms": 0.201917, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "U5n5YN7gaSE=", + "parent_span_id": "jB0/2W4MDhw=", + "duration_ms": 0.168542, + "status": {} + }, + { + "service": "messenger-api", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "vy0IHJh/3iY=", + "parent_span_id": "jB0/2W4MDhw=", + "duration_ms": 5.372708, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "span_id": "2i2vmcn9EE0=", + "parent_span_id": "jB0/2W4MDhw=", + "duration_ms": 0.102708, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "T3Dr7iQuvIQ=", + "parent_span_id": "jB0/2W4MDhw=", + "duration_ms": 0.4535, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "w3sVwhZtutg=", + "parent_span_id": "jB0/2W4MDhw=", + "duration_ms": 0.147042, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "jLqcbPc5vVk=", + "parent_span_id": "jB0/2W4MDhw=", + "duration_ms": 0.211042, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "vRHP/KuDBrM=", + "parent_span_id": "jB0/2W4MDhw=", + "duration_ms": 0.248459, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "RRs6tYlWUvQ=", + "parent_span_id": "jB0/2W4MDhw=", + "duration_ms": 0.422917, + "status": {} + }, + { + "service": "messenger-dispatcher", + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "span_id": "LNSmERepEgY=", + "parent_span_id": "jB0/2W4MDhw=", + "duration_ms": 8.300958, + "status": {} + }, + { + "service": "messenger-worker", + "name": "attachments consume", + "kind": "SPAN_KIND_CONSUMER", + "span_id": "IvTbWoSGVoE=", + "parent_span_id": "LNSmERepEgY=", + "duration_ms": 31.394333, + "status": {} + }, + { + "service": "messenger-worker", + "name": "attachment.process", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "4tlyqCkXguI=", + "parent_span_id": "IvTbWoSGVoE=", + "duration_ms": 31.374083, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "span_id": "Izvr4bX032o=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.06025, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "zFMrsz5K56k=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.078375, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "1Frlsrq2oUE=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.075459, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "PfWiD/Xq8zY=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.070958, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "FYsBFGWK6dU=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.082667, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "KdQxJRcUJIs=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.170958, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.get", + "kind": "SPAN_KIND_CLIENT", + "span_id": "LIlqz9FL5z8=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.715417, + "status": {} + }, + { + "service": "messenger-worker", + "name": "image.transform", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "xOom63W1sGk=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 20.176666, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "fwgrCS52cB8=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 1.659, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "vPvf2s/AvEk=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 1.142, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "ir1VprbWua0=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 1.498667, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "L3Dc7HDuWxE=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 1.126917, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "span_id": "JrHiDoPaSdY=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.062458, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "7lXqP9/QjPc=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.09775, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "0M+RM0MUW84=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.131875, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "B4dOvJxvPNI=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.137917, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "WgHS/xPupz0=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.070083, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "wVzdIKgGGh4=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.045916, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "S113KWQ30yo=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.048042, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "5jgU8/Rdz6o=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.058, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "+1GTkh/cu7o=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.115833, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "JlJYBMU/6sU=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.052959, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "NeM1XkL62a0=", + "parent_span_id": "4tlyqCkXguI=", + "duration_ms": 0.240375, + "status": {} + } + ], + "diagnosis": "Worker resumes consumption and publishes S3/DB results" + }, + "db_long_1": { + "captured": 1794819076.086566, + "observed": { + "at": 1794819070.248044, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "8.669241743836409e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "4.072512647554806e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 20.0, + "queue_age": 0.026991, + "api_burn": 86.69241743836409, + "ready_burn": 0.0 + }, + "trace_id": "70b67102a6f3d7ee869684371ce298e4", + "spans": [ + { + "service": "messenger-api", + "name": "GET /api/v1/chats/{chat_id}/messages", + "kind": "SPAN_KIND_SERVER", + "span_id": "l4LW33+CV8U=", + "parent_span_id": null, + "duration_ms": 2004.636376, + "status": { + "message": "server_error", + "code": "STATUS_CODE_ERROR" + } + }, + { + "service": "messenger-api", + "name": "chat.ListForChat", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "FQlPCLoE8vo=", + "parent_span_id": "l4LW33+CV8U=", + "duration_ms": 1995.711834, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "tTpmp+AqbYg=", + "parent_span_id": "FQlPCLoE8vo=", + "duration_ms": 1995.446126, + "status": { + "message": "operation_failed", + "code": "STATUS_CODE_ERROR" + } + }, + { + "service": "messenger-api", + "name": "auth.VerifySession", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "UKhXjQIOvRo=", + "parent_span_id": "l4LW33+CV8U=", + "duration_ms": 8.668708, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "Tc5NlVoaWJY=", + "parent_span_id": "UKhXjQIOvRo=", + "duration_ms": 1.714875, + "status": {} + }, + { + "service": "messenger-api", + "name": "auth.validateStoredSession", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "C8ovABCKlJg=", + "parent_span_id": "UKhXjQIOvRo=", + "duration_ms": 0.003292, + "status": {} + }, + { + "service": "messenger-api", + "name": "chat.GetForMember", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "KBOMYXXvers=", + "parent_span_id": "FQlPCLoE8vo=", + "duration_ms": 0.217958, + "status": {} + }, + { + "service": "messenger-api", + "name": "chat.cachedChat", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "gAtGRRQt9jc=", + "parent_span_id": "KBOMYXXvers=", + "duration_ms": 0.21, + "status": {} + } + ], + "error_log_events": 100, + "diagnosis": "DB wait dominates server span; HTTP 5xx, not image processing" + }, + "worker_long_2": { + "captured": 1794820117.7003589, + "observed": { + "at": 1794820114.816596, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "3.80952380952381e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "6.1957976e+01" + } + ], + "pending": 34.0, + "db_lock_waiting": 0.0, + "queue_age": 67.099234, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + "worker_received_events": 0, + "accepted_upload_events": 36, + "diagnosis": "Queue age and ready burn increase while API accepts uploads and no worker consumption is observed" + }, + "worker_recovery_2": { + "captured": 1794820303.223746, + "trace_id": "330cb47d547a48641c18b3585d67ac72", + "spans": [ + { + "service": "messenger-api", + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "span_id": "k/gMZFk1iaE=", + "parent_span_id": null, + "duration_ms": 16.477959, + "status": {} + }, + { + "service": "messenger-api", + "name": "auth.VerifySession", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "J3VpnmW6ENA=", + "parent_span_id": "k/gMZFk1iaE=", + "duration_ms": 0.598792, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "ov7Xujq7i7U=", + "parent_span_id": "J3VpnmW6ENA=", + "duration_ms": 0.448584, + "status": {} + }, + { + "service": "messenger-api", + "name": "auth.validateStoredSession", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "C8q0/ZrIe/4=", + "parent_span_id": "J3VpnmW6ENA=", + "duration_ms": 0.004, + "status": {} + }, + { + "service": "messenger-api", + "name": "attachment.upload", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "wcpec0H5X1g=", + "parent_span_id": "k/gMZFk1iaE=", + "duration_ms": 15.75775, + "status": {} + }, + { + "service": "messenger-api", + "name": "chat.GetForMember", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "rDQHGB0JB6o=", + "parent_span_id": "wcpec0H5X1g=", + "duration_ms": 0.440792, + "status": {} + }, + { + "service": "messenger-api", + "name": "chat.cachedChat", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "pn5u0G3Ll7Y=", + "parent_span_id": "rDQHGB0JB6o=", + "duration_ms": 0.392958, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "8JJI9w/pVGQ=", + "parent_span_id": "wcpec0H5X1g=", + "duration_ms": 0.204875, + "status": {} + }, + { + "service": "messenger-api", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "lOtQ3UnY3j4=", + "parent_span_id": "wcpec0H5X1g=", + "duration_ms": 3.791291, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "span_id": "WjoYRErEAJE=", + "parent_span_id": "wcpec0H5X1g=", + "duration_ms": 0.133209, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "vjTSfMzkxdY=", + "parent_span_id": "wcpec0H5X1g=", + "duration_ms": 0.494125, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "iSlRUK8WnBk=", + "parent_span_id": "wcpec0H5X1g=", + "duration_ms": 0.224916, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "gB0niK2Bqdc=", + "parent_span_id": "wcpec0H5X1g=", + "duration_ms": 0.286167, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "AxJsDmtC51U=", + "parent_span_id": "wcpec0H5X1g=", + "duration_ms": 0.355041, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "/ZJ+mkhNErk=", + "parent_span_id": "wcpec0H5X1g=", + "duration_ms": 0.574, + "status": {} + }, + { + "service": "messenger-dispatcher", + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "span_id": "5+Cb1uWFrpE=", + "parent_span_id": "wcpec0H5X1g=", + "duration_ms": 4.772875, + "status": {} + }, + { + "service": "messenger-worker", + "name": "attachments consume", + "kind": "SPAN_KIND_CONSUMER", + "span_id": "4cXaN795nBY=", + "parent_span_id": "5+Cb1uWFrpE=", + "duration_ms": 72.435459, + "status": {} + }, + { + "service": "messenger-worker", + "name": "attachment.process", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "EBDmTXukus8=", + "parent_span_id": "4cXaN795nBY=", + "duration_ms": 72.412792, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "span_id": "Qaftqwf3pMA=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.056541, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "vbij5a+7aU0=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.082125, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "BbyE/aQY9Do=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.092125, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "kKb22VVz8mA=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.09925, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "KIQUjBb56B4=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.111541, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "OAIEFOqUexQ=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.194417, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.get", + "kind": "SPAN_KIND_CLIENT", + "span_id": "tFQQx0YRJR0=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.872166, + "status": {} + }, + { + "service": "messenger-worker", + "name": "image.transform", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "XN/tZ15MVDU=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 61.892666, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "3IDDcvD0Ie8=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 1.696125, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "MxYsC1zK4Ak=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.778375, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "t/TDuVclitk=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.631208, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "jBWSxFHarC0=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 1.008625, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "span_id": "zaNX6x89OG8=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.031959, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "LOKdHjLdseU=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.060833, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "BrvxeG3Bnj0=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.053292, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "/+hO+OZZZTU=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.113083, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "2N0Nv4wQ1KE=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.077542, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "RCiSYYwRIFw=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.097958, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "G97CVA2rxr0=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.104417, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "hZkdFAFAg60=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.103625, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "4rjobgveC8Q=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.14575, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "UOlESjsLd8o=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.089375, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "WIZAFfmopB4=", + "parent_span_id": "EBDmTXukus8=", + "duration_ms": 0.262916, + "status": {} + } + ], + "diagnosis": "Worker resumes consumption and publishes S3/DB results" + }, + "db_long_2": { + "captured": 1794821004.6703832, + "observed": { + "at": 1794821003.013938, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.245960502692998e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "4.335016835016835e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 92.45960502692998, + "ready_burn": 0.0 + }, + "trace_id": "dba5ec05cea91c4f71ef86ea40f5c1d6", + "spans": [ + { + "service": "messenger-api", + "name": "GET /api/v1/chats/{chat_id}/messages", + "kind": "SPAN_KIND_SERVER", + "span_id": "W8DTh3gzEN4=", + "parent_span_id": null, + "duration_ms": 2000.358335, + "status": { + "message": "server_error", + "code": "STATUS_CODE_ERROR" + } + }, + { + "service": "messenger-api", + "name": "chat.ListForChat", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "NgHeWhn4j8A=", + "parent_span_id": "W8DTh3gzEN4=", + "duration_ms": 1999.084626, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "Y03GQ6O63PQ=", + "parent_span_id": "NgHeWhn4j8A=", + "duration_ms": 1998.919459, + "status": { + "message": "operation_failed", + "code": "STATUS_CODE_ERROR" + } + }, + { + "service": "messenger-api", + "name": "auth.VerifySession", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "XZlAv4egFP8=", + "parent_span_id": "W8DTh3gzEN4=", + "duration_ms": 1.182416, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "Tt2/p7PJ7Ec=", + "parent_span_id": "XZlAv4egFP8=", + "duration_ms": 1.085709, + "status": {} + }, + { + "service": "messenger-api", + "name": "auth.validateStoredSession", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "/82g86zbTBI=", + "parent_span_id": "XZlAv4egFP8=", + "duration_ms": 0.003, + "status": {} + }, + { + "service": "messenger-api", + "name": "chat.GetForMember", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "fnLz8ACRdIQ=", + "parent_span_id": "NgHeWhn4j8A=", + "duration_ms": 0.145042, + "status": {} + }, + { + "service": "messenger-api", + "name": "chat.cachedChat", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "cwM1CvmO6Lw=", + "parent_span_id": "fnLz8ACRdIQ=", + "duration_ms": 0.137792, + "status": {} + } + ], + "error_log_events": 100, + "diagnosis": "DB wait dominates server span; HTTP 5xx, not image processing" + }, + "worker_long_3": { + "captured": 1794822046.653913, + "observed": { + "at": 1794822039.5664508, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "3.0952380952380953e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "5.543069e+01" + } + ], + "pending": 31.0, + "db_lock_waiting": 0.0, + "queue_age": 60.215234, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + "worker_received_events": 0, + "accepted_upload_events": 34, + "diagnosis": "Queue age and ready burn increase while API accepts uploads and no worker consumption is observed" + }, + "worker_recovery_3": { + "captured": 1794822237.1867661, + "trace_id": "fc776e4ce81ed1e4bec2a01ddbf3cb0d", + "spans": [ + { + "service": "messenger-api", + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "span_id": "7TDhdOt3JWE=", + "parent_span_id": null, + "duration_ms": 15.748125, + "status": {} + }, + { + "service": "messenger-api", + "name": "auth.VerifySession", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "Y09nMtaHj3A=", + "parent_span_id": "7TDhdOt3JWE=", + "duration_ms": 0.944875, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "wA3Tp2tq28w=", + "parent_span_id": "Y09nMtaHj3A=", + "duration_ms": 0.553167, + "status": {} + }, + { + "service": "messenger-api", + "name": "auth.validateStoredSession", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "Ti2CzIJpy2s=", + "parent_span_id": "Y09nMtaHj3A=", + "duration_ms": 0.004667, + "status": {} + }, + { + "service": "messenger-api", + "name": "attachment.upload", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "WK1gaBFFVuE=", + "parent_span_id": "7TDhdOt3JWE=", + "duration_ms": 14.667834, + "status": {} + }, + { + "service": "messenger-api", + "name": "chat.GetForMember", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "CinB8l5DWyA=", + "parent_span_id": "WK1gaBFFVuE=", + "duration_ms": 0.207042, + "status": {} + }, + { + "service": "messenger-api", + "name": "chat.cachedChat", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "MPLNDjPFTGs=", + "parent_span_id": "CinB8l5DWyA=", + "duration_ms": 0.195583, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "gLAsHguQKJs=", + "parent_span_id": "WK1gaBFFVuE=", + "duration_ms": 0.176125, + "status": {} + }, + { + "service": "messenger-api", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "slPqX/PjJvc=", + "parent_span_id": "WK1gaBFFVuE=", + "duration_ms": 3.282292, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "span_id": "aWLEza5FoNc=", + "parent_span_id": "WK1gaBFFVuE=", + "duration_ms": 0.18775, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "cwSwxxVuG+o=", + "parent_span_id": "WK1gaBFFVuE=", + "duration_ms": 0.356625, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "h3XnJ6fEaP8=", + "parent_span_id": "WK1gaBFFVuE=", + "duration_ms": 0.142208, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "GOSwmhBifjA=", + "parent_span_id": "WK1gaBFFVuE=", + "duration_ms": 0.215291, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "BZIl8s0D6Ko=", + "parent_span_id": "WK1gaBFFVuE=", + "duration_ms": 0.6495, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "Ucj2+IHycyk=", + "parent_span_id": "WK1gaBFFVuE=", + "duration_ms": 1.128209, + "status": {} + }, + { + "service": "messenger-dispatcher", + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "span_id": "TUTjKL0aoQw=", + "parent_span_id": "WK1gaBFFVuE=", + "duration_ms": 3.255125, + "status": {} + }, + { + "service": "messenger-worker", + "name": "attachments consume", + "kind": "SPAN_KIND_CONSUMER", + "span_id": "RLDGQG6V4n0=", + "parent_span_id": "TUTjKL0aoQw=", + "duration_ms": 34.185083, + "status": {} + }, + { + "service": "messenger-worker", + "name": "attachment.process", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "Pn1t6GmqDek=", + "parent_span_id": "RLDGQG6V4n0=", + "duration_ms": 34.163542, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "span_id": "MmRYJySAS28=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.090458, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "vA1/uiZwbWI=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.094834, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "2tkz18ZRkSQ=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.087292, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "SxxMfEhz9NQ=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.081292, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "YDxvts+aBr0=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.09875, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "WsgKZ0xgywo=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.191625, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.get", + "kind": "SPAN_KIND_CLIENT", + "span_id": "2LYtJzkun6k=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.961208, + "status": {} + }, + { + "service": "messenger-worker", + "name": "image.transform", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "TQoQle9dCjU=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 22.377, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "hBZvR/WFYNU=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 2.531291, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "KofZ6u0pSUs=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 1.005625, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "ZdCSweVtcaE=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 1.100958, + "status": {} + }, + { + "service": "messenger-worker", + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "span_id": "avO/NoIryEM=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.817125, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "span_id": "nQAVTXQSYQw=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.08125, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "QJpJIT3r0n4=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.113333, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "DIFxbHI1mCM=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.079125, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "NTder7igcDg=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.099167, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "Pw1J28uULFc=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.0895, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "pfmXdAen7Lk=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.054375, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "NpXP0D72q7k=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.045, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "XyU72PKv/1o=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.06825, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "u2/4v+rFM4g=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.112833, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "span_id": "PfO4wiFa5xY=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.116083, + "status": {} + }, + { + "service": "messenger-worker", + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "xMZGbigLVN4=", + "parent_span_id": "Pn1t6GmqDek=", + "duration_ms": 0.255875, + "status": {} + } + ], + "diagnosis": "Worker resumes consumption and publishes S3/DB results" + }, + "db_long_3": { + "captured": 1794822938.670801, + "observed": { + "at": 1794822934.982975, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.529616724738676e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "4.6043771043771037e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.29616724738676, + "ready_burn": 0.0 + }, + "trace_id": "41c627868dc842ca5fc041495bc21a04", + "spans": [ + { + "service": "messenger-api", + "name": "GET /api/v1/chats/{chat_id}/messages", + "kind": "SPAN_KIND_SERVER", + "span_id": "uzhM6DftApw=", + "parent_span_id": null, + "duration_ms": 2001.004543, + "status": { + "message": "server_error", + "code": "STATUS_CODE_ERROR" + } + }, + { + "service": "messenger-api", + "name": "chat.ListForChat", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "54LdqkseDrI=", + "parent_span_id": "uzhM6DftApw=", + "duration_ms": 1997.504459, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "6UWBiEmdS04=", + "parent_span_id": "54LdqkseDrI=", + "duration_ms": 1997.362043, + "status": { + "message": "operation_failed", + "code": "STATUS_CODE_ERROR" + } + }, + { + "service": "messenger-api", + "name": "auth.VerifySession", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "24sh8rlWUcs=", + "parent_span_id": "uzhM6DftApw=", + "duration_ms": 3.384833, + "status": {} + }, + { + "service": "messenger-api", + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "span_id": "3CinSlCF1os=", + "parent_span_id": "24sh8rlWUcs=", + "duration_ms": 0.878083, + "status": {} + }, + { + "service": "messenger-api", + "name": "auth.validateStoredSession", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "RNEb85RT04M=", + "parent_span_id": "24sh8rlWUcs=", + "duration_ms": 0.00275, + "status": {} + }, + { + "service": "messenger-api", + "name": "chat.GetForMember", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "0Sxa6sK/ELY=", + "parent_span_id": "54LdqkseDrI=", + "duration_ms": 0.125083, + "status": {} + }, + { + "service": "messenger-api", + "name": "chat.cachedChat", + "kind": "SPAN_KIND_INTERNAL", + "span_id": "SJYElvHcN5w=", + "parent_span_id": "0Sxa6sK/ELY=", + "duration_ms": 0.119375, + "status": {} + } + ], + "error_log_events": 100, + "diagnosis": "DB wait dominates server span; HTTP 5xx, not image processing" + } + }, + "collection_errors": [] +} diff --git a/evidence/loki-config.txt b/evidence/loki-config.txt new file mode 100644 index 0000000..a5ff221 --- /dev/null +++ b/evidence/loki-config.txt @@ -0,0 +1,3 @@ + Container mtusi2026lab6-loki-run-e631f6d38697 Creating + Container mtusi2026lab6-loki-run-e631f6d38697 Created +level=info ts=2026-11-16T07:28:10.950532589Z caller=main.go:109 msg="config is valid" diff --git a/evidence/persistence-process-checks.txt b/evidence/persistence-process-checks.txt new file mode 100644 index 0000000..0962286 --- /dev/null +++ b/evidence/persistence-process-checks.txt @@ -0,0 +1 @@ +PASS: persistence, real restart of all stores, unchanged volume identities diff --git a/evidence/persistence-processes.json b/evidence/persistence-processes.json new file mode 100644 index 0000000..36c32c3 --- /dev/null +++ b/evidence/persistence-processes.json @@ -0,0 +1,220 @@ +{ + "started": 1794824173.232328, + "finished": 1794824240.646864, + "command": [ + "make", + "test-persistence", + "ACTION_SCRIPT=scripts/restart.sh" + ], + "returncode": 0, + "before": { + "mtusi2026lab6-api-1": { + "service": "api", + "started_at": "2026-11-16T08:13:21.61203226Z", + "volumes": [] + }, + "mtusi2026lab6-api-2": { + "service": "api", + "started_at": "2026-11-16T08:13:21.47365851Z", + "volumes": [] + }, + "mtusi2026lab6-broker-1": { + "service": "broker", + "started_at": "2026-11-16T07:26:13.19346959Z", + "volumes": [ + "mtusi2026lab6_broker_data" + ] + }, + "mtusi2026lab6-dispatcher-1": { + "service": "dispatcher", + "started_at": "2026-11-16T08:13:21.48129851Z", + "volumes": [] + }, + "mtusi2026lab6-grafana-1": { + "service": "grafana", + "started_at": "2026-11-16T07:17:01.172943543Z", + "volumes": [ + "mtusi2026lab6_grafana_data" + ] + }, + "mtusi2026lab6-lb-1": { + "service": "lb", + "started_at": "2026-11-16T07:18:04.666655461Z", + "volumes": [] + }, + "mtusi2026lab6-loki-1": { + "service": "loki", + "started_at": "2026-11-16T07:28:11.851867589Z", + "volumes": [ + "mtusi2026lab6_loki_data" + ] + }, + "mtusi2026lab6-otel-collector-1": { + "service": "otel-collector", + "started_at": "2026-11-16T07:17:00.733510959Z", + "volumes": [] + }, + "mtusi2026lab6-postgres-1": { + "service": "postgres", + "started_at": "2026-11-16T07:26:11.526101381Z", + "volumes": [ + "mtusi2026lab6_postgres_data" + ] + }, + "mtusi2026lab6-prometheus-1": { + "service": "prometheus", + "started_at": "2026-11-16T07:17:00.662441876Z", + "volumes": [ + "mtusi2026lab6_prometheus_data" + ] + }, + "mtusi2026lab6-redis-1": { + "service": "redis", + "started_at": "2026-11-16T07:26:11.457109339Z", + "volumes": [ + "mtusi2026lab6_redis_data" + ] + }, + "mtusi2026lab6-s3-1": { + "service": "s3", + "started_at": "2026-11-16T07:50:16.345342466Z", + "volumes": [ + "mtusi2026lab6_s3_data" + ] + }, + "mtusi2026lab6-tempo-1": { + "service": "tempo", + "started_at": "2026-11-16T10:14:35.984164211Z", + "volumes": [ + "mtusi2026lab6_tempo_data" + ] + }, + "mtusi2026lab6-worker-1": { + "service": "worker", + "started_at": "2026-11-16T09:43:34.761728669Z", + "volumes": [] + } + }, + "after": { + "mtusi2026lab6-api-1": { + "service": "api", + "started_at": "2026-11-16T10:17:15.204445174Z", + "volumes": [] + }, + "mtusi2026lab6-api-2": { + "service": "api", + "started_at": "2026-11-16T10:17:15.085021257Z", + "volumes": [] + }, + "mtusi2026lab6-broker-1": { + "service": "broker", + "started_at": "2026-11-16T10:16:17.026791924Z", + "volumes": [ + "mtusi2026lab6_broker_data" + ] + }, + "mtusi2026lab6-dispatcher-1": { + "service": "dispatcher", + "started_at": "2026-11-16T10:17:15.074397715Z", + "volumes": [] + }, + "mtusi2026lab6-grafana-1": { + "service": "grafana", + "started_at": "2026-11-16T10:16:16.370235216Z", + "volumes": [ + "mtusi2026lab6_grafana_data" + ] + }, + "mtusi2026lab6-lb-1": { + "service": "lb", + "started_at": "2026-11-16T10:17:15.316601507Z", + "volumes": [] + }, + "mtusi2026lab6-loki-1": { + "service": "loki", + "started_at": "2026-11-16T10:16:18.125747508Z", + "volumes": [ + "mtusi2026lab6_loki_data" + ] + }, + "mtusi2026lab6-otel-collector-1": { + "service": "otel-collector", + "started_at": "2026-11-16T10:16:17.9785623Z", + "volumes": [] + }, + "mtusi2026lab6-postgres-1": { + "service": "postgres", + "started_at": "2026-11-16T10:16:16.097257841Z", + "volumes": [ + "mtusi2026lab6_postgres_data" + ] + }, + "mtusi2026lab6-prometheus-1": { + "service": "prometheus", + "started_at": "2026-11-16T10:16:15.930341549Z", + "volumes": [ + "mtusi2026lab6_prometheus_data" + ] + }, + "mtusi2026lab6-redis-1": { + "service": "redis", + "started_at": "2026-11-16T10:16:16.023978757Z", + "volumes": [ + "mtusi2026lab6_redis_data" + ] + }, + "mtusi2026lab6-s3-1": { + "service": "s3", + "started_at": "2026-11-16T10:16:41.23677863Z", + "volumes": [ + "mtusi2026lab6_s3_data" + ] + }, + "mtusi2026lab6-tempo-1": { + "service": "tempo", + "started_at": "2026-11-16T10:16:16.187121549Z", + "volumes": [ + "mtusi2026lab6_tempo_data" + ] + }, + "mtusi2026lab6-worker-1": { + "service": "worker", + "started_at": "2026-11-16T10:17:15.092656715Z", + "volumes": [] + } + }, + "checks": { + "postgres": { + "restarted": true, + "same_volumes": true + }, + "redis": { + "restarted": true, + "same_volumes": true + }, + "s3": { + "restarted": true, + "same_volumes": true + }, + "broker": { + "restarted": true, + "same_volumes": true + }, + "prometheus": { + "restarted": true, + "same_volumes": true + }, + "loki": { + "restarted": true, + "same_volumes": true + }, + "tempo": { + "restarted": true, + "same_volumes": true + }, + "grafana": { + "restarted": true, + "same_volumes": true + } + } +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..b138d03 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18086" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/pipeline-checks.txt b/evidence/pipeline-checks.txt new file mode 100644 index 0000000..f1c555f --- /dev/null +++ b/evidence/pipeline-checks.txt @@ -0,0 +1,13 @@ +{"scenario": "worker_down_and_store_restart", "run": 1, "result": "PASS", "seconds": 66.526, "attachment_id": "2c3b8f0c-8104-4619-a1fe-8041c09a3beb", "job_id": "668e23d6-b840-4577-ba49-3b90791fc3ec", "postgres_restarted": true} +{"scenario": "crash_before_objects", "run": 1, "result": "PASS", "seconds": 11.223, "job_id": "41e04582-a1bc-4c99-be90-1f927df9903b", "manifest_rows": 4} +{"scenario": "crash_after_original_put", "run": 1, "result": "PASS", "seconds": 10.999, "job_id": "515ac833-e1e4-46dc-96ab-fdcb89e7ed40", "manifest_rows": 4} +{"scenario": "crash_after_objects", "run": 1, "result": "PASS", "seconds": 10.992, "job_id": "aceb0f8b-3084-4569-b562-d80cc11f58e2", "manifest_rows": 4} +{"scenario": "crash_after_result_commit", "run": 1, "result": "PASS", "seconds": 1.992, "job_id": "f0694c6f-9b0b-41e1-98f4-5816d52c860e", "manifest_rows": 4} +{"scenario": "crash_before_ack", "run": 1, "result": "PASS", "seconds": 2.052, "job_id": "8609eafe-72b6-4640-bf8d-447e8fa8eb89", "manifest_rows": 4} +{"scenario": "ready_redelivery", "run": 1, "result": "PASS", "seconds": 2.585, "job_id": "c741a727-32ed-4846-878c-be719aa5bc51", "copies": 3} +{"scenario": "crash_confirm_before_outbox_mark", "run": 1, "result": "PASS", "seconds": 34.324, "job_id": "22b742c2-f101-4f6d-965a-0dad4bc4a287"} +{"scenario": "lease_reorder_and_stale_version", "run": 1, "result": "PASS", "seconds": 14.869, "attachment_id": "bcd403eb-854f-4ea9-b0f2-272731468061", "old_job_id": "e930a58f-f6f0-43cd-b2c0-860cec77ff33", "pipeline_version": 2, "manifest_rows": 8} +{"scenario": "bounded_retry_and_explicit_replay", "run": 1, "result": "PASS", "seconds": 13.693, "job_id": "dd8d36b6-387f-453e-a898-e08b7674523e", "failed_attempt": 5, "replayed_attempt": 6} +{"scenario": "permanent_error", "run": 1, "result": "PASS", "seconds": 2.667, "job_id": "83a5bdb8-7e1a-4b33-ba40-557ec119c534"} +{"scenario": "api_crash_after_staging", "run": 1, "result": "PASS", "seconds": 0.149, "attachment_id": "328ad979-0ee9-4d41-8b16-7c420d4173f0", "accepted_commit": false} +{"scenario": "api_crash_after_upload_commit", "run": 1, "result": "PASS", "seconds": 2.262, "attachment_id": "cc0a1cf1-862e-4e81-a276-449b075f5c48", "accepted_commit": true} diff --git a/evidence/pipeline.json b/evidence/pipeline.json new file mode 100644 index 0000000..59e564a --- /dev/null +++ b/evidence/pipeline.json @@ -0,0 +1,123 @@ +{ + "date_utc": "2026-11-16T10:20:50.742339+00:00", + "commit": "8b381c115209cf7ac143df4826c468bf38128d71", + "working_tree": "implementation changes uncommitted", + "repeats": 1, + "worker": { + "concurrency": 2, + "prefetch": 2, + "timeout_seconds": 5, + "lease_seconds": 8, + "memory": "1GiB", + "cpus": 1 + }, + "results": [ + { + "scenario": "worker_down_and_store_restart", + "run": 1, + "result": "PASS", + "seconds": 66.526, + "attachment_id": "2c3b8f0c-8104-4619-a1fe-8041c09a3beb", + "job_id": "668e23d6-b840-4577-ba49-3b90791fc3ec", + "postgres_restarted": true + }, + { + "scenario": "crash_before_objects", + "run": 1, + "result": "PASS", + "seconds": 11.223, + "job_id": "41e04582-a1bc-4c99-be90-1f927df9903b", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_original_put", + "run": 1, + "result": "PASS", + "seconds": 10.999, + "job_id": "515ac833-e1e4-46dc-96ab-fdcb89e7ed40", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_objects", + "run": 1, + "result": "PASS", + "seconds": 10.992, + "job_id": "aceb0f8b-3084-4569-b562-d80cc11f58e2", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_result_commit", + "run": 1, + "result": "PASS", + "seconds": 1.992, + "job_id": "f0694c6f-9b0b-41e1-98f4-5816d52c860e", + "manifest_rows": 4 + }, + { + "scenario": "crash_before_ack", + "run": 1, + "result": "PASS", + "seconds": 2.052, + "job_id": "8609eafe-72b6-4640-bf8d-447e8fa8eb89", + "manifest_rows": 4 + }, + { + "scenario": "ready_redelivery", + "run": 1, + "result": "PASS", + "seconds": 2.585, + "job_id": "c741a727-32ed-4846-878c-be719aa5bc51", + "copies": 3 + }, + { + "scenario": "crash_confirm_before_outbox_mark", + "run": 1, + "result": "PASS", + "seconds": 34.324, + "job_id": "22b742c2-f101-4f6d-965a-0dad4bc4a287" + }, + { + "scenario": "lease_reorder_and_stale_version", + "run": 1, + "result": "PASS", + "seconds": 14.869, + "attachment_id": "bcd403eb-854f-4ea9-b0f2-272731468061", + "old_job_id": "e930a58f-f6f0-43cd-b2c0-860cec77ff33", + "pipeline_version": 2, + "manifest_rows": 8 + }, + { + "scenario": "bounded_retry_and_explicit_replay", + "run": 1, + "result": "PASS", + "seconds": 13.693, + "job_id": "dd8d36b6-387f-453e-a898-e08b7674523e", + "failed_attempt": 5, + "replayed_attempt": 6 + }, + { + "scenario": "permanent_error", + "run": 1, + "result": "PASS", + "seconds": 2.667, + "job_id": "83a5bdb8-7e1a-4b33-ba40-557ec119c534" + }, + { + "scenario": "api_crash_after_staging", + "run": 1, + "result": "PASS", + "seconds": 0.149, + "attachment_id": "328ad979-0ee9-4d41-8b16-7c420d4173f0", + "accepted_commit": false + }, + { + "scenario": "api_crash_after_upload_commit", + "run": 1, + "result": "PASS", + "seconds": 2.262, + "attachment_id": "cc0a1cf1-862e-4e81-a276-449b075f5c48", + "accepted_commit": true + } + ], + "image_id": "sha256:c7021c3fed2700b5234fedad71dd0478acb126a697aa25ec5ac1a2c99020962b" +} diff --git a/evidence/previous-lab-checks.txt b/evidence/previous-lab-checks.txt new file mode 100644 index 0000000..492faaf --- /dev/null +++ b/evidence/previous-lab-checks.txt @@ -0,0 +1,6 @@ +python3 student-tests/images.py +PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs +python3 student-tests/storage.py +PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified +python3 student-tests/broker_persistence.py +PASS: confirmed message survived broker container replacement with the same volume and stable node identity diff --git a/evidence/process-events.json b/evidence/process-events.json new file mode 100644 index 0000000..3222593 --- /dev/null +++ b/evidence/process-events.json @@ -0,0 +1,65 @@ +[ + { + "time": 1794814090673707380, + "action": "create", + "service": "loki", + "container": "c2c4608c32287a2666869a56b5d11cab57bbcc23268d46b5927b3df895b1a4c0", + "exit_code": null + }, + { + "time": 1794814090753412339, + "action": "start", + "service": "loki", + "container": "c2c4608c32287a2666869a56b5d11cab57bbcc23268d46b5927b3df895b1a4c0", + "exit_code": null + }, + { + "time": 1794814091040505797, + "action": "die", + "service": "loki", + "container": "c2c4608c32287a2666869a56b5d11cab57bbcc23268d46b5927b3df895b1a4c0", + "exit_code": "0" + }, + { + "time": 1794814091079755672, + "action": "destroy", + "service": "loki", + "container": "c2c4608c32287a2666869a56b5d11cab57bbcc23268d46b5927b3df895b1a4c0", + "exit_code": null + }, + { + "time": 1794814091245157881, + "action": "kill", + "service": "loki", + "container": "421f16b0cf6f897f24ffc81e9f95e16919b0364882475923c15de7b8bbfa8e41", + "exit_code": null + }, + { + "time": 1794814091841150214, + "action": "stop", + "service": "loki", + "container": "421f16b0cf6f897f24ffc81e9f95e16919b0364882475923c15de7b8bbfa8e41", + "exit_code": null + }, + { + "time": 1794814091842610839, + "action": "die", + "service": "loki", + "container": "421f16b0cf6f897f24ffc81e9f95e16919b0364882475923c15de7b8bbfa8e41", + "exit_code": "0" + }, + { + "time": 1794814091940223131, + "action": "start", + "service": "loki", + "container": "421f16b0cf6f897f24ffc81e9f95e16919b0364882475923c15de7b8bbfa8e41", + "exit_code": null + }, + { + "time": 1794814091940331339, + "action": "restart", + "service": "loki", + "container": "421f16b0cf6f897f24ffc81e9f95e16919b0364882475923c15de7b8bbfa8e41", + "exit_code": null + } +] diff --git a/evidence/prometheus-config.txt b/evidence/prometheus-config.txt new file mode 100644 index 0000000..b5b8b05 --- /dev/null +++ b/evidence/prometheus-config.txt @@ -0,0 +1,7 @@ +Checking /etc/prometheus/prometheus.yaml + SUCCESS: 1 rule files found + SUCCESS: /etc/prometheus/prometheus.yaml is valid prometheus config file syntax + +Checking /etc/prometheus/alerts.yaml + SUCCESS: 16 rules found + diff --git a/evidence/provisioning.json b/evidence/provisioning.json new file mode 100644 index 0000000..49e8673 --- /dev/null +++ b/evidence/provisioning.json @@ -0,0 +1,52 @@ +{ + "dashboard_uid": "messenger-lab6", + "panels": 22, + "provisioned": true, + "datasources": [ + { + "uid": "loki", + "type": "loki", + "jsonData": { + "derivedFields": [ + { + "datasourceUid": "tempo", + "matcherRegex": "trace_id", + "matcherType": "label", + "name": "TraceID", + "url": "${__value.raw}" + } + ], + "maxLines": 1000 + } + }, + { + "uid": "prometheus", + "type": "prometheus", + "jsonData": { + "timeInterval": "5s" + } + }, + { + "uid": "tempo", + "type": "tempo", + "jsonData": { + "nodeGraph": { + "enabled": true + }, + "tracesToLogsV2": { + "datasourceUid": "loki", + "filterBySpanID": false, + "filterByTraceID": true, + "spanEndTimeShift": "1m", + "spanStartTimeShift": "-1m", + "tags": [ + { + "key": "service.name", + "value": "service_name" + } + ] + } + } + } + ] +} diff --git a/evidence/research-validation.json b/evidence/research-validation.json new file mode 100644 index 0000000..548975e --- /dev/null +++ b/evidence/research-validation.json @@ -0,0 +1,12 @@ +{ + "research_completed": true, + "short_incidents": 6, + "short_incidents_with_firing": 0, + "long_incidents": 6, + "long_incidents_all_expected_alerts_fired_and_resolved": true, + "collection_errors": 0, + "generator_dropped": 0, + "confirmed_messages_missing": 0, + "attachments_ready": 3529, + "attachments_accepted": 3529 +} diff --git a/evidence/restore.txt b/evidence/restore.txt new file mode 100644 index 0000000..e8a6f93 --- /dev/null +++ b/evidence/restore.txt @@ -0,0 +1,33 @@ +./scripts/compose.sh up -d --wait --wait-timeout 90 --no-recreate api lb + Container mtusi2026lab6-s3-1 Running + Container mtusi2026lab6-postgres-1 Running + Container mtusi2026lab6-api-1 Running + Container mtusi2026lab6-lb-1 Running + Container mtusi2026lab6-s3-1 Waiting + Container mtusi2026lab6-postgres-1 Waiting + Container mtusi2026lab6-postgres-1 Healthy + Container mtusi2026lab6-s3-1 Healthy + Container mtusi2026lab6-migrate-1 Starting + Container mtusi2026lab6-s3-init-1 Starting + Container mtusi2026lab6-s3-init-1 Started + Container mtusi2026lab6-migrate-1 Started + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-api-2 Starting + Container mtusi2026lab6-api-2 Started + Container mtusi2026lab6-lb-1 Waiting + Container mtusi2026lab6-s3-1 Waiting + Container mtusi2026lab6-postgres-1 Waiting + Container mtusi2026lab6-migrate-1 Waiting + Container mtusi2026lab6-api-2 Waiting + Container mtusi2026lab6-api-1 Waiting + Container mtusi2026lab6-s3-init-1 Waiting + Container mtusi2026lab6-s3-init-1 Exited + Container mtusi2026lab6-postgres-1 Healthy + Container mtusi2026lab6-s3-1 Healthy + Container mtusi2026lab6-migrate-1 Exited + Container mtusi2026lab6-lb-1 Healthy + Container mtusi2026lab6-api-2 Healthy + Container mtusi2026lab6-api-1 Healthy diff --git a/evidence/signal-checks.txt b/evidence/signal-checks.txt new file mode 100644 index 0000000..282180b --- /dev/null +++ b/evidence/signal-checks.txt @@ -0,0 +1 @@ +PASS: stored metrics, safe labels, privacy canaries, trace search and dashboard queries diff --git a/evidence/signals.json b/evidence/signals.json new file mode 100644 index 0000000..468918c --- /dev/null +++ b/evidence/signals.json @@ -0,0 +1,3720 @@ +{ + "timestamp": 1794906730.467176, + "queries": { + "messenger_http_requests_total": [ + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "DELETE", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions/current", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "DELETE", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions/current", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "2" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "DELETE", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions/current", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "DELETE", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions/current", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "DELETE", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions/current", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "10" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}/content", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}/content", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "4" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}/content", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}/content", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "2" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}/content", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "2" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "2" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/users/me", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/users/me", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "2" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/users/me", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/users/me", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "4" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/users/me", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/live", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/live", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "3" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/live", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/live", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/live", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/ready", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/ready", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "21731" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/ready", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/ready", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/ready", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "12" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "16" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "7" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/attachments", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/attachments", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "8" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/attachments", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/attachments", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "7" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/attachments", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "85" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "13" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/users", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/users", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "26" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/users", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/users", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "3" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/users", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "DELETE", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions/current", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "DELETE", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions/current", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "DELETE", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions/current", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "DELETE", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions/current", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "DELETE", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions/current", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "10" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "2" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}/content", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}/content", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "2" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}/content", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}/content", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/attachments/{attachment_id}/content", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "2" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "9" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "2" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/users/me", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/users/me", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "2" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/users/me", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/users/me", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "4" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/api/v1/users/me", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/live", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/live", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "3" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/live", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/live", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/live", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/ready", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/ready", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "21734" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/ready", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/ready", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "GET", + "otel_scope_name": "messenger", + "route": "/health/ready", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "6" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "26" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/auth/sessions", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "9" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/attachments", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/attachments", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "4" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/attachments", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/attachments", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "10" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/attachments", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "84" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "14" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/chats/{chat_id}/messages", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/users", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "1xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/users", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "2xx" + }, + "value": [ + 1794906730.47, + "16" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/users", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "3xx" + }, + "value": [ + 1794906730.47, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/users", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "4xx" + }, + "value": [ + 1794906730.47, + "3" + ] + }, + { + "metric": { + "__name__": "messenger_http_requests_total", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "method": "POST", + "otel_scope_name": "messenger", + "route": "/api/v1/users", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6", + "status_class": "5xx" + }, + "value": [ + 1794906730.47, + "0" + ] + } + ], + "messenger_jobs_total": [ + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "2104ce28-e28c-4101-af4e-f0968f884d99", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "duplicate", + "service_instance_id": "2104ce28-e28c-4101-af4e-f0968f884d99", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.475, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "2104ce28-e28c-4101-af4e-f0968f884d99", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "failed", + "service_instance_id": "2104ce28-e28c-4101-af4e-f0968f884d99", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.475, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "2104ce28-e28c-4101-af4e-f0968f884d99", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "ready", + "service_instance_id": "2104ce28-e28c-4101-af4e-f0968f884d99", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.475, + "3" + ] + }, + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "2104ce28-e28c-4101-af4e-f0968f884d99", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "retry", + "service_instance_id": "2104ce28-e28c-4101-af4e-f0968f884d99", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.475, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "298b76de-9c77-4689-9614-e2e055e8abd8", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "duplicate", + "service_instance_id": "298b76de-9c77-4689-9614-e2e055e8abd8", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.475, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "298b76de-9c77-4689-9614-e2e055e8abd8", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "failed", + "service_instance_id": "298b76de-9c77-4689-9614-e2e055e8abd8", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.475, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "298b76de-9c77-4689-9614-e2e055e8abd8", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "ready", + "service_instance_id": "298b76de-9c77-4689-9614-e2e055e8abd8", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.475, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "298b76de-9c77-4689-9614-e2e055e8abd8", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "retry", + "service_instance_id": "298b76de-9c77-4689-9614-e2e055e8abd8", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.475, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "4b7b1dd7-3543-4844-b15e-638f72579703", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "duplicate", + "service_instance_id": "4b7b1dd7-3543-4844-b15e-638f72579703", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.475, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "4b7b1dd7-3543-4844-b15e-638f72579703", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "failed", + "service_instance_id": "4b7b1dd7-3543-4844-b15e-638f72579703", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.475, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "4b7b1dd7-3543-4844-b15e-638f72579703", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "ready", + "service_instance_id": "4b7b1dd7-3543-4844-b15e-638f72579703", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.475, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_jobs_total", + "deployment_environment_name": "lab", + "exported_instance": "4b7b1dd7-3543-4844-b15e-638f72579703", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "retry", + "service_instance_id": "4b7b1dd7-3543-4844-b15e-638f72579703", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.475, + "0" + ] + } + ], + "messenger_job_duration_seconds_count": [ + { + "metric": { + "__name__": "messenger_job_duration_seconds_count", + "deployment_environment_name": "lab", + "exported_instance": "2104ce28-e28c-4101-af4e-f0968f884d99", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "ready", + "service_instance_id": "2104ce28-e28c-4101-af4e-f0968f884d99", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.479, + "3" + ] + }, + { + "metric": { + "__name__": "messenger_job_duration_seconds_count", + "deployment_environment_name": "lab", + "exported_instance": "298b76de-9c77-4689-9614-e2e055e8abd8", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "ready", + "service_instance_id": "298b76de-9c77-4689-9614-e2e055e8abd8", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.479, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_job_duration_seconds_count", + "deployment_environment_name": "lab", + "exported_instance": "298b76de-9c77-4689-9614-e2e055e8abd8", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "retry", + "service_instance_id": "298b76de-9c77-4689-9614-e2e055e8abd8", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.479, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_job_duration_seconds_count", + "deployment_environment_name": "lab", + "exported_instance": "4b7b1dd7-3543-4844-b15e-638f72579703", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "outcome": "ready", + "service_instance_id": "4b7b1dd7-3543-4844-b15e-638f72579703", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.479, + "1" + ] + } + ], + "messenger_job_ready_duration_seconds_count": [ + { + "metric": { + "__name__": "messenger_job_ready_duration_seconds_count", + "deployment_environment_name": "lab", + "exported_instance": "2104ce28-e28c-4101-af4e-f0968f884d99", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "service_instance_id": "2104ce28-e28c-4101-af4e-f0968f884d99", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.521, + "3" + ] + }, + { + "metric": { + "__name__": "messenger_job_ready_duration_seconds_count", + "deployment_environment_name": "lab", + "exported_instance": "298b76de-9c77-4689-9614-e2e055e8abd8", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "service_instance_id": "298b76de-9c77-4689-9614-e2e055e8abd8", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.521, + "1" + ] + }, + { + "metric": { + "__name__": "messenger_job_ready_duration_seconds_count", + "deployment_environment_name": "lab", + "exported_instance": "4b7b1dd7-3543-4844-b15e-638f72579703", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "job_type": "attachment.process.v1", + "otel_scope_name": "messenger", + "service_instance_id": "4b7b1dd7-3543-4844-b15e-638f72579703", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.521, + "1" + ] + } + ], + "messenger_db_pool_used": [ + { + "metric": { + "__name__": "messenger_db_pool_used", + "deployment_environment_name": "lab", + "exported_instance": "68226416-722e-4f43-87dd-214422250ce8", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "68226416-722e-4f43-87dd-214422250ce8", + "service_name": "messenger-dispatcher", + "service_version": "lab6" + }, + "value": [ + 1794906730.526, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_db_pool_used", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794906730.526, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_db_pool_used", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794906730.526, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_db_pool_used", + "deployment_environment_name": "lab", + "exported_instance": "2104ce28-e28c-4101-af4e-f0968f884d99", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "2104ce28-e28c-4101-af4e-f0968f884d99", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.526, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_db_pool_used", + "deployment_environment_name": "lab", + "exported_instance": "298b76de-9c77-4689-9614-e2e055e8abd8", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "298b76de-9c77-4689-9614-e2e055e8abd8", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.526, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_db_pool_used", + "deployment_environment_name": "lab", + "exported_instance": "4b7b1dd7-3543-4844-b15e-638f72579703", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "4b7b1dd7-3543-4844-b15e-638f72579703", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.526, + "1" + ] + } + ], + "messenger_worker_busy": [ + { + "metric": { + "__name__": "messenger_worker_busy", + "deployment_environment_name": "lab", + "exported_instance": "68226416-722e-4f43-87dd-214422250ce8", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "68226416-722e-4f43-87dd-214422250ce8", + "service_name": "messenger-dispatcher", + "service_version": "lab6" + }, + "value": [ + 1794906730.53, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_worker_busy", + "deployment_environment_name": "lab", + "exported_instance": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "d9dce648-8f79-4be8-bcb7-d1395b825a1e", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794906730.53, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_worker_busy", + "deployment_environment_name": "lab", + "exported_instance": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "exported_job": "messenger-api", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "f202ca94-9d49-45e1-886d-4721eb4e0f21", + "service_name": "messenger-api", + "service_version": "lab6" + }, + "value": [ + 1794906730.53, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_worker_busy", + "deployment_environment_name": "lab", + "exported_instance": "2104ce28-e28c-4101-af4e-f0968f884d99", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "2104ce28-e28c-4101-af4e-f0968f884d99", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.53, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_worker_busy", + "deployment_environment_name": "lab", + "exported_instance": "298b76de-9c77-4689-9614-e2e055e8abd8", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "298b76de-9c77-4689-9614-e2e055e8abd8", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.53, + "0" + ] + }, + { + "metric": { + "__name__": "messenger_worker_busy", + "deployment_environment_name": "lab", + "exported_instance": "4b7b1dd7-3543-4844-b15e-638f72579703", + "exported_job": "messenger-worker", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "4b7b1dd7-3543-4844-b15e-638f72579703", + "service_name": "messenger-worker", + "service_version": "lab6" + }, + "value": [ + 1794906730.53, + "0" + ] + } + ], + "messenger_queue_depth": [ + { + "metric": { + "__name__": "messenger_queue_depth", + "deployment_environment_name": "lab", + "exported_instance": "68226416-722e-4f43-87dd-214422250ce8", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments.v1", + "service_instance_id": "68226416-722e-4f43-87dd-214422250ce8", + "service_name": "messenger-dispatcher", + "service_version": "lab6" + }, + "value": [ + 1794906730.535, + "0" + ] + } + ], + "messenger_queue_oldest_age_seconds": [ + { + "metric": { + "__name__": "messenger_queue_oldest_age_seconds", + "deployment_environment_name": "lab", + "exported_instance": "68226416-722e-4f43-87dd-214422250ce8", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "queue": "attachments.v1", + "service_instance_id": "68226416-722e-4f43-87dd-214422250ce8", + "service_name": "messenger-dispatcher", + "service_version": "lab6" + }, + "value": [ + 1794906730.539, + "0" + ] + } + ], + "messenger_ready_slo_total": [ + { + "metric": { + "__name__": "messenger_ready_slo_total", + "deployment_environment_name": "lab", + "exported_instance": "68226416-722e-4f43-87dd-214422250ce8", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "outcome": "bad", + "service_instance_id": "68226416-722e-4f43-87dd-214422250ce8", + "service_name": "messenger-dispatcher", + "service_version": "lab6" + }, + "value": [ + 1794906730.543, + "422" + ] + }, + { + "metric": { + "__name__": "messenger_ready_slo_total", + "deployment_environment_name": "lab", + "exported_instance": "68226416-722e-4f43-87dd-214422250ce8", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "outcome": "good", + "service_instance_id": "68226416-722e-4f43-87dd-214422250ce8", + "service_name": "messenger-dispatcher", + "service_version": "lab6" + }, + "value": [ + 1794906730.543, + "4251" + ] + } + ], + "messenger_observation_timestamp": [ + { + "metric": { + "__name__": "messenger_observation_timestamp", + "deployment_environment_name": "lab", + "exported_instance": "68226416-722e-4f43-87dd-214422250ce8", + "exported_job": "messenger-dispatcher", + "instance": "otel-collector:8889", + "job": "messenger", + "otel_scope_name": "messenger", + "service_instance_id": "68226416-722e-4f43-87dd-214422250ce8", + "service_name": "messenger-dispatcher", + "service_version": "lab6" + }, + "value": [ + 1794906730.548, + "1794906725" + ] + } + ], + "ALERTS": [], + "otelcol_exporter_send_failed_spans_total": [ + { + "metric": { + "__name__": "otelcol_exporter_send_failed_spans_total", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794906730.558, + "0" + ] + } + ], + "otelcol_exporter_sent_spans_total": [ + { + "metric": { + "__name__": "otelcol_exporter_sent_spans_total", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794906730.563, + "495192" + ] + } + ], + "otelcol_receiver_accepted_spans_total": [ + { + "metric": { + "__name__": "otelcol_receiver_accepted_spans_total", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/receiver/receiverhelper", + "receiver": "otlp", + "transport": "http" + }, + "value": [ + 1794906730.567, + "495196" + ] + } + ] + }, + "checks": { + "required_metrics_present": true, + "forbidden_labels_absent": true, + "raw_uuid_routes_absent": true, + "synthetic_canary_absent_in_logs": true, + "synthetic_canary_absent_in_traces": true, + "dashboard_expressions_valid": true + }, + "series_count": 987, + "trace_ids_checked": [ + "ccb8136805e2c5cf018a90efc0b4b4", + "1162feb5dbd368a2913f33b570a9b23", + "d1529ce18cc0c51f59d3ad9aaf141f", + "997cfd081206b85ec5233f09b19123", + "e479bcd8cfe7b4c02b0a54a64d0039", + "3adb9f461a3647984fb21f5b8da06ce", + "2a2ae05942df366991e2bfacadc0a4a", + "3f506df6d0646c7edc3212fb413b96", + "ddae810e9ace6b81f9d12f68a66ea4", + "1cb7b4cbfa63468caff85bfb642ddad" + ] +} diff --git a/evidence/slo-experiment.json b/evidence/slo-experiment.json new file mode 100644 index 0000000..28a21ee --- /dev/null +++ b/evidence/slo-experiment.json @@ -0,0 +1,39965 @@ +{ + "started": 1794817078.754661, + "revision": "470d9d6f1e4fe4f594e37c2ba59550ee5ca54fc1", + "parameters": { + "warmup": 900, + "long": 240, + "cooldown": 660, + "repeats": 3, + "backend": 300, + "output": "evidence/slo-experiment.json" + }, + "profile": { + "message_rps": 10, + "read_fraction": 0.9, + "image_interval_seconds": 2, + "max_inflight": 64, + "transparent_retries": false + }, + "events": [ + { + "phase": "baseline", + "at": 1794817079.719438 + }, + { + "phase": "worker_short_1", + "at": 1794817980.084564 + }, + { + "phase": "worker_short_recovery_1", + "at": 1794817985.306079 + }, + { + "phase": "db_short_1", + "at": 1794818045.434122 + }, + { + "phase": "db_short_recovery_1", + "at": 1794818050.568737 + }, + { + "phase": "worker_long_1", + "at": 1794818110.87646 + }, + { + "phase": "worker_recovery_1", + "at": 1794818351.155176 + }, + { + "phase": "db_long_1", + "at": 1794819011.315764 + }, + { + "phase": "db_recovery_1", + "at": 1794819251.457268 + }, + { + "phase": "worker_short_2", + "at": 1794819911.837276 + }, + { + "phase": "worker_short_recovery_2", + "at": 1794819917.066106 + }, + { + "phase": "db_short_2", + "at": 1794819977.215701 + }, + { + "phase": "db_short_recovery_2", + "at": 1794819982.3592389 + }, + { + "phase": "worker_long_2", + "at": 1794820042.693995 + }, + { + "phase": "worker_recovery_2", + "at": 1794820282.981466 + }, + { + "phase": "db_long_2", + "at": 1794820943.135081 + }, + { + "phase": "db_recovery_2", + "at": 1794821183.300124 + }, + { + "phase": "worker_short_3", + "at": 1794821843.637631 + }, + { + "phase": "worker_short_recovery_3", + "at": 1794821848.894581 + }, + { + "phase": "db_short_3", + "at": 1794821909.053349 + }, + { + "phase": "db_short_recovery_3", + "at": 1794821914.218946 + }, + { + "phase": "worker_long_3", + "at": 1794821974.6094139 + }, + { + "phase": "worker_recovery_3", + "at": 1794822214.850919 + }, + { + "phase": "db_long_3", + "at": 1794822875.0330508 + }, + { + "phase": "db_recovery_3", + "at": 1794823115.229683 + }, + { + "phase": "tempo_unavailable", + "at": 1794823775.759885 + }, + { + "phase": "tempo_recovery", + "at": 1794824076.114967 + }, + { + "phase": "final_drain", + "at": 1794824136.201777 + } + ], + "observations": [ + { + "at": 1794817079.719736, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": null, + "ready_burn": null + }, + { + "at": 1794817086.289414, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817091.326719, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817096.3617432, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817101.402138, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817106.4349902, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817111.476392, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817118.2910678, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817123.325936, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817128.359777, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817133.3935602, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817138.447664, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817143.491947, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817150.665087, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817155.7148118, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817160.76382, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817165.80141, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817170.842396, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817175.8865972, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817183.042644, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817188.087595, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817193.127789, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817198.16373, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817203.201547, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.066741, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817208.238318, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817215.4334831, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817220.472054, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817225.5177238, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817230.563076, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817235.6078, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817240.646415, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817247.277921, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817252.326472, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817257.368146, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817262.420275, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817267.4678679, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817272.513035, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817279.7044659, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817284.747232, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817289.783363, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817294.8165429, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817299.853066, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817304.891766, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817312.0674691, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817317.110316, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817322.157366, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817327.197133, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817332.237809, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817337.275027, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817344.4819882, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817349.520747, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817354.5647008, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817359.5991528, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817364.626428, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817369.6641312, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817376.26107, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817381.292151, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817386.328467, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817391.3584719, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817396.389138, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817401.4425359, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817408.2916338, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817413.339564, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817418.382522, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817423.429079, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817428.474408, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817433.5308821, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817440.702518, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817445.753493, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817450.7946692, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.055262, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817455.838173, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817460.877528, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817465.922198, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817473.0769331, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817478.1229608, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817483.1552901, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817488.1942668, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817493.2318652, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817498.2739341, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817505.474976, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817510.510591, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817515.56227, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817520.601204, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817525.641185, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817530.686151, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817537.306885, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817542.340959, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817547.3954198, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817552.4336698, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817557.488605, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817562.5494552, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817569.712182, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817574.7441542, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817579.782659, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817584.823742, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817589.870603, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817594.907205, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817602.102602, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817607.1600819, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817612.218507, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817617.254758, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817622.298521, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817627.3575602, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817634.558194, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817639.595063, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817644.6293728, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817649.6788852, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817654.713015, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817659.754653, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817666.340294, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817671.394558, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817676.430323, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817681.4657001, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817686.511447, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817691.562412, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817698.355316, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817703.398145, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817708.43926, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817713.507437, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817718.548935, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817723.5811698, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817730.7757719, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817735.810185, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817740.8521118, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817745.900019, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817750.939741, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817755.9764638, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817763.13458, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817768.182821, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817773.219808, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817778.262992, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817783.320907, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.015206, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817788.3553028, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817795.611884, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817800.660275, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817805.701856, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817810.744863, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817815.784086, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817820.822664, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817827.367137, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817832.402153, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817837.455861, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817842.509994, + "phase": "baseline", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.032573, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817847.55895, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817852.6140192, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817859.820897, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817864.862689, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817869.902797, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817874.935694, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817879.986324, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817885.022869, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817892.1976802, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817897.2464252, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817902.2949681, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817907.348644, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817912.390671, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817917.445576, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817924.652884, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817929.693032, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817934.750782, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817939.7978492, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817944.859468, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817949.902196, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817956.432082, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817961.476749, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817966.5230231, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817971.561867, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817976.60252, + "phase": "baseline", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817980.09135, + "phase": "worker_short_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817985.3123462, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817992.4855092, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794817997.5419898, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818002.587258, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.068471, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818007.641813, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818012.6760871, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818017.711661, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818024.899025, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818029.933311, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818034.974436, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818040.015083, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818045.079872, + "phase": "worker_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818045.440297, + "phase": "db_short_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818050.5758011, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 2.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818057.4475121, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818062.486887, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.161667282162244, + "ready_burn": 0.0 + }, + { + "at": 1794818067.526314, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.161659060672179, + "ready_burn": 0.0 + }, + { + "at": 1794818072.561645, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.161667282162244, + "ready_burn": 0.0 + }, + { + "at": 1794818077.609184, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.161650839180687, + "ready_burn": 0.0 + }, + { + "at": 1794818082.661085, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.170623839219398, + "ready_burn": 0.0 + }, + { + "at": 1794818089.840401, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.161667282162244, + "ready_burn": 0.0 + }, + { + "at": 1794818094.8918452, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.161659060672179, + "ready_burn": 0.0 + }, + { + "at": 1794818099.9413528, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.161667282162245, + "ready_burn": 0.0 + }, + { + "at": 1794818104.990975, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.161667282162244, + "ready_burn": 0.0 + }, + { + "at": 1794818110.0242612, + "phase": "db_short_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.065067858060367, + "ready_burn": 0.0 + }, + { + "at": 1794818110.884144, + "phase": "worker_long_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.065067858060367, + "ready_burn": 0.0 + }, + { + "at": 1794818117.411634, + "phase": "worker_long_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818122.452794, + "phase": "worker_long_1", + "alerts": [], + "pending": 3.0, + "db_lock_waiting": 0.0, + "queue_age": 4.304007, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818127.4945889, + "phase": "worker_long_1", + "alerts": [], + "pending": 5.0, + "db_lock_waiting": 0.0, + "queue_age": 9.508749, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818132.540808, + "phase": "worker_long_1", + "alerts": [], + "pending": 8.0, + "db_lock_waiting": 0.0, + "queue_age": 14.724927, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818137.603219, + "phase": "worker_long_1", + "alerts": [], + "pending": 10.0, + "db_lock_waiting": 0.0, + "queue_age": 18.834138, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818142.6588092, + "phase": "worker_long_1", + "alerts": [], + "pending": 13.0, + "db_lock_waiting": 0.0, + "queue_age": 24.037828, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818149.8231401, + "phase": "worker_long_1", + "alerts": [], + "pending": 18.0, + "db_lock_waiting": 0.0, + "queue_age": 34.442132, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818154.865221, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "3.4442132e+01" + } + ], + "pending": 20.0, + "db_lock_waiting": 0.0, + "queue_age": 39.624112, + "api_burn": 0.0, + "ready_burn": 23.076923076923077 + }, + { + "at": 1794818159.91112, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "3.9624112e+01" + } + ], + "pending": 22.0, + "db_lock_waiting": 0.0, + "queue_age": 43.761888, + "api_burn": 0.0, + "ready_burn": 41.666666666666664 + }, + { + "at": 1794818164.946391, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "4.3761888e+01" + } + ], + "pending": 25.0, + "db_lock_waiting": 0.0, + "queue_age": 49.031502, + "api_burn": 0.0, + "ready_burn": 58.333333333333336 + }, + { + "at": 1794818169.9908311, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "7.692307692307693e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "2.3255813953488372e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "4.9031502e+01" + } + ], + "pending": 28.0, + "db_lock_waiting": 0.0, + "queue_age": 54.168141, + "api_burn": 0.0, + "ready_burn": 76.92307692307692 + }, + { + "at": 1794818175.0395598, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "3.023255813953488e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "5.4168141e+01" + } + ], + "pending": 30.0, + "db_lock_waiting": 0.0, + "queue_age": 59.309107, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818182.217382, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "3.5714285714285715e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "5.9309107e+01" + } + ], + "pending": 33.0, + "db_lock_waiting": 0.0, + "queue_age": 64.524647, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818187.2704482, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "4.186046511627907e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "6.4524647e+01" + } + ], + "pending": 35.0, + "db_lock_waiting": 0.0, + "queue_age": 69.653883, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818192.320253, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "4.761904761904762e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "6.9653883e+01" + } + ], + "pending": 37.0, + "db_lock_waiting": 0.0, + "queue_age": 73.825973, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818197.364158, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "5.238095238095238e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "7.3825973e+01" + } + ], + "pending": 40.0, + "db_lock_waiting": 0.0, + "queue_age": 79.032641, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818202.413971, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "5.813953488372093e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "7.9032641e+01" + } + ], + "pending": 43.0, + "db_lock_waiting": 0.0, + "queue_age": 84.199718, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818207.460534, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "6.511627906976745e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "8.4199718e+01" + } + ], + "pending": 45.0, + "db_lock_waiting": 0.0, + "queue_age": 89.40145, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818214.655318, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "7.142857142857143e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "8.940145e+01" + } + ], + "pending": 50.0, + "db_lock_waiting": 0.0, + "queue_age": 99.743737, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818219.698456, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "7.674418604651163e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "9.4554534e+01" + } + ], + "pending": 52.0, + "db_lock_waiting": 0.0, + "queue_age": 103.930069, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818224.738105, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "8.80952380952381e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.03930069e+02" + } + ], + "pending": 55.0, + "db_lock_waiting": 0.0, + "queue_age": 109.071457, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818229.792893, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "9.302325581395349e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.09071457e+02" + } + ], + "pending": 58.0, + "db_lock_waiting": 0.0, + "queue_age": 114.225677, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818234.827488, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.14225677e+02" + } + ], + "pending": 60.0, + "db_lock_waiting": 0.0, + "queue_age": 119.436457, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818239.877002, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.19436457e+02" + } + ], + "pending": 63.0, + "db_lock_waiting": 0.0, + "queue_age": 124.725895, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818246.463666, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.24725895e+02" + } + ], + "pending": 65.0, + "db_lock_waiting": 0.0, + "queue_age": 128.834976, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818251.522675, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.28834976e+02" + } + ], + "pending": 67.0, + "db_lock_waiting": 0.0, + "queue_age": 133.976725, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818256.56956, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.33976725e+02" + } + ], + "pending": 70.0, + "db_lock_waiting": 0.0, + "queue_age": 139.179279, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818261.609104, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.39179279e+02" + } + ], + "pending": 73.0, + "db_lock_waiting": 0.0, + "queue_age": 144.33553, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818266.649786, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.4433553e+02" + } + ], + "pending": 75.0, + "db_lock_waiting": 0.0, + "queue_age": 149.53104, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818271.700781, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.4953104e+02" + } + ], + "pending": 78.0, + "db_lock_waiting": 0.0, + "queue_age": 154.746014, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818278.446375, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.54746014e+02" + } + ], + "pending": 80.0, + "db_lock_waiting": 0.0, + "queue_age": 158.953274, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818283.493334, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.58953274e+02" + } + ], + "pending": 83.0, + "db_lock_waiting": 0.0, + "queue_age": 164.082883, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818288.545137, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.64082883e+02" + } + ], + "pending": 85.0, + "db_lock_waiting": 0.0, + "queue_age": 169.291162, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818293.593587, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.69291162e+02" + } + ], + "pending": 88.0, + "db_lock_waiting": 0.0, + "queue_age": 174.521674, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818298.634774, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.74521674e+02" + } + ], + "pending": 90.0, + "db_lock_waiting": 0.0, + "queue_age": 179.729801, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818303.6781912, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.79729801e+02" + } + ], + "pending": 92.0, + "db_lock_waiting": 0.0, + "queue_age": 183.843441, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818310.855326, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.8906473e+02" + } + ], + "pending": 98.0, + "db_lock_waiting": 0.0, + "queue_age": 194.224767, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818315.9351912, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.94224767e+02" + } + ], + "pending": 100.0, + "db_lock_waiting": 0.0, + "queue_age": 199.503384, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818320.97957, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "1.99503384e+02" + } + ], + "pending": 103.0, + "db_lock_waiting": 0.0, + "queue_age": 204.724013, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818326.036243, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "2.04724013e+02" + } + ], + "pending": 105.0, + "db_lock_waiting": 0.0, + "queue_age": 208.84019, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818331.098156, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "2.0884019e+02" + } + ], + "pending": 107.0, + "db_lock_waiting": 0.0, + "queue_age": 213.974741, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818336.152699, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "2.13974741e+02" + } + ], + "pending": 110.0, + "db_lock_waiting": 0.0, + "queue_age": 219.186804, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818343.325943, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "2.19186804e+02" + } + ], + "pending": 113.0, + "db_lock_waiting": 0.0, + "queue_age": 224.342443, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818348.387014, + "phase": "worker_long_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "2.24342443e+02" + } + ], + "pending": 115.0, + "db_lock_waiting": 0.0, + "queue_age": 229.536154, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818351.165162, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "2.29536154e+02" + } + ], + "pending": 118.0, + "db_lock_waiting": 0.0, + "queue_age": 234.743949, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818357.45053, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "2.34743949e+02" + } + ], + "pending": 120.0, + "db_lock_waiting": 0.0, + "queue_age": 238.945361, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818362.4843059, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T08:35:49.758725496Z", + "value": "2.38945361e+02" + } + ], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.072361, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794818367.52706, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "6.428571428571428e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "7.916666666666667e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 64.28571428571428 + }, + { + "at": 1794818372.584648, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "5.714285714285714e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "7.534246575342466e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 57.14285714285714 + }, + { + "at": 1794818377.6392698, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "5.116279069767442e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "7.123287671232876e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 51.16279069767442 + }, + { + "at": 1794818382.683199, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "4.5238095238095233e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "6.805555555555556e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 45.238095238095234 + }, + { + "at": 1794818389.857276, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "3.488372093023256e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "6.11111111111111e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 34.883720930232556 + }, + { + "at": 1794818394.904481, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "2.7906976744186046e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "5.753424657534246e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 27.906976744186046 + }, + { + "at": 1794818399.942503, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "2.142857142857143e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "5.416666666666666e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 21.42857142857143 + }, + { + "at": 1794818404.980233, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1.627906976744186e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "5.068493150684931e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 16.279069767441857 + }, + { + "at": 1794818410.031358, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "9.523809523809523e-02" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "4.7222222222222227e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 9.523809523809524 + }, + { + "at": 1794818415.075768, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "4.444444444444444e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 4.761904761904762 + }, + { + "at": 1794818422.245017, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "4.109589041095891e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818427.290702, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "3.6986301369863017e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818432.3332639, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "3.333333333333333e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818437.39058, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "3.013698630136986e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818442.427396, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "2.638888888888889e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818447.470419, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "2.361111111111111e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818454.671204, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "2.0547945205479454e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818459.704594, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "1.6438356164383564e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818464.745909, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "9.589041095890412e-02" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818469.783495, + "phase": "worker_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:36:04.758725496Z", + "value": "5.555555555555556e-02" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818474.821053, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818479.858977, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818486.4775732, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818491.530831, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818496.565601, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818501.608222, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818506.6413288, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818511.696003, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818518.482427, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818523.547446, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818528.5968928, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818533.650391, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818538.70134, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818543.744302, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818550.924813, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818555.985779, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818561.037513, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818566.095542, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818571.1410298, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818576.198688, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818583.3896298, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818588.4430232, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818593.497683, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818598.5372741, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818603.58408, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818608.6323462, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818615.859988, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818620.909682, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818625.9629052, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818631.016671, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818636.077066, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818641.114092, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818647.503802, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818652.568862, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818657.609641, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818662.650371, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.036384, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818667.7005188, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818672.759617, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818679.956577, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818685.006716, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818690.063858, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818695.114722, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818700.1533198, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818705.204672, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818712.386878, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818717.4437509, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818722.490601, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.060171, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818727.5449228, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818732.593954, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818737.630663, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818744.835239, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818749.872104, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818754.933093, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818759.982951, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818765.0338728, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818770.0750751, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818776.480602, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818781.5379992, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818786.5801349, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818791.630319, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818796.663403, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818801.702892, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.093155, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818808.5312588, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818813.571924, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818818.6139991, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818823.666589, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818828.719254, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818833.7646248, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818840.920732, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818845.972718, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818851.0046709, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818856.059531, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818861.097281, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818866.155497, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818873.343261, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818878.39594, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818883.44223, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818888.4828339, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818893.536503, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818898.586376, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818905.799886, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818910.840498, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818915.894124, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818920.937928, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818925.991227, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818931.0324411, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818937.567403, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818942.625279, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818947.6720462, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818952.7253928, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818957.773149, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818962.8196118, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818969.9745321, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.020243, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818975.009193, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818980.050816, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818985.0932539, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818990.144236, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794818995.1945899, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819002.39566, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819007.4371, + "phase": "worker_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819011.329417, + "phase": "db_long_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819017.608098, + "phase": "db_long_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819022.663584, + "phase": "db_long_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819027.720171, + "phase": "db_long_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 5.9245960502693, + "ready_burn": 0.0 + }, + { + "at": 1794819032.766957, + "phase": "db_long_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 15.080789946140035, + "ready_burn": 0.0 + }, + { + "at": 1794819037.816284, + "phase": "db_long_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 23.877917414721725, + "ready_burn": 0.0 + }, + { + "at": 1794819042.866842, + "phase": "db_long_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 33.03411131059246, + "ready_burn": 0.0 + }, + { + "at": 1794819050.040202, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "5.080789946140036e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "2.3861720067453623e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 50.80789946140036, + "ready_burn": 0.0 + }, + { + "at": 1794819055.082208, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "5.978456014362656e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "2.803030303030303e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 19.0, + "queue_age": 0.0, + "api_burn": 59.784560143626564, + "ready_burn": 0.0 + }, + { + "at": 1794819060.140764, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "6.875223826310575e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "3.229342327150085e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 68.75223826310575, + "ready_burn": 0.0 + }, + { + "at": 1794819065.189108, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "7.771612172101267e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "3.647851727042966e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 77.71612172101267, + "ready_burn": 0.0 + }, + { + "at": 1794819070.248044, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "8.669241743836409e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "4.072512647554806e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 20.0, + "queue_age": 0.026991, + "api_burn": 86.69241743836409, + "ready_burn": 0.0 + }, + { + "at": 1794819075.293143, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.499723984619848e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "4.4903117101937656e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 94.99723984619848, + "ready_burn": 0.0 + }, + { + "at": 1794819082.488804, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "4.915682967959527e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794819087.535007, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.514731369150781e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "5.332771693344567e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.1473136915078, + "ready_burn": 0.0 + }, + { + "at": 1794819092.591435, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "5.75885328836425e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794819097.64719, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.515570934256057e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "6.17833134919424e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.15570934256057, + "ready_burn": 0.0 + }, + { + "at": 1794819102.702792, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "6.601703335067687e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794819107.739028, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "7.019441241778238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794819114.92942, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "7.860715930785234e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794819119.979671, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "8.280110142645754e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794819125.019596, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "8.702535193574904e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794819130.077763, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.122910217632119e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794819135.129369, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.512829413411337e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794819140.175423, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.53287197231834e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.528026822980871e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3287197231834, + "ready_burn": 0.0 + }, + { + "at": 1794819146.566572, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794819151.617321, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794819156.659818, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.516407599309155e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.519867549668873e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 19.0, + "queue_age": 0.0, + "api_burn": 95.16407599309154, + "ready_burn": 0.0 + }, + { + "at": 1794819161.698252, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794819166.744659, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.520264681555004e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794819171.795913, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 20.0, + "queue_age": 0.044872, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794819178.5808818, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.520264681555004e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794819183.62434, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794819188.68477, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794819193.7421691, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794819198.800565, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.519867549668873e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794819203.8363142, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 19.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794819211.009229, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.527754763877382e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794819216.0594492, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.514731369150781e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.1473136915078, + "ready_burn": 0.0 + }, + { + "at": 1794819221.106941, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.527754763877382e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794819226.151017, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.514731369150781e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.1473136915078, + "ready_burn": 0.0 + }, + { + "at": 1794819231.199438, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.531249999999999e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.527363184079601e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.31249999999999, + "ready_burn": 0.0 + }, + { + "at": 1794819236.2470138, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.519867549668873e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794819243.448579, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.527363184079602e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794819248.501982, + "phase": "db_long_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794819251.472235, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.532062391681108e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.527363184079601e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681107, + "ready_burn": 0.0 + }, + { + "at": 1794819258.651328, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.519469759734879e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794819263.704313, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.179229480737017e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "9.35562805872757e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 91.79229480737017, + "ready_burn": 0.0 + }, + { + "at": 1794819268.750327, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "8.324958123953099e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "8.940505297473513e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 83.24958123953098, + "ready_burn": 0.0 + }, + { + "at": 1794819273.7953029, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "7.504187604690117e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "8.539967373572593e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 75.04187604690117, + "ready_burn": 0.0 + }, + { + "at": 1794819278.8425741, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "6.638795986622074e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "8.118892508143322e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 66.38795986622074, + "ready_burn": 0.0 + }, + { + "at": 1794819283.892714, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "5.829145728643216e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "7.724306688417618e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 58.291457286432156, + "ready_burn": 0.0 + }, + { + "at": 1794819291.059426, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "3.30550918196995e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "6.490228013029316e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 33.0550918196995, + "ready_burn": 0.0 + }, + { + "at": 1794819296.106236, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "3.30550918196995e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "6.490228013029316e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 33.0550918196995, + "ready_burn": 0.0 + }, + { + "at": 1794819301.151285, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "2.479061976549414e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "6.096169519152405e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.026248, + "api_burn": 24.79061976549414, + "ready_burn": 0.0 + }, + { + "at": 1794819306.200667, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "1.636060100166945e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "5.675895765472312e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 16.36060100166945, + "ready_burn": 0.0 + }, + { + "at": 1794819311.246229, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "8.040201005025126e-02" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "5.28117359413203e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 8.040201005025125, + "ready_burn": 0.0 + }, + { + "at": 1794819316.2936509, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "4.8615635179153094e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819323.482641, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "4.466177669111654e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819328.527766, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "4.047231270358306e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819333.572733, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "3.6482084690553745e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819338.618208, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "3.2328990228013027e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819343.676069, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "2.833876221498371e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819348.7361422, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "2.42473555736371e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819355.927978, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "1.61106590724166e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819360.9761002, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "1.2052117263843647e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819366.025252, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "7.973962571196096e-02" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819371.07248, + "phase": "db_recovery_1", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T08:50:39.758725496Z", + "value": "3.908794788273615e-02" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819376.124644, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819381.178331, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819387.631534, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819392.6952472, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819397.762194, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819402.809151, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819407.8497438, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819412.890559, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819420.082487, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819425.129048, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819430.1884532, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819435.237744, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819440.28267, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819445.323009, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819452.565143, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819457.614454, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819462.6723242, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819467.723307, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819472.78406, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819477.8352058, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819485.0246139, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819490.066738, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819495.120152, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819500.1773388, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819505.2224839, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819510.261987, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819516.619497, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819521.662731, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819526.701041, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819531.758432, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819536.8090649, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819541.875551, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819548.6489742, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819553.711732, + "phase": "db_recovery_1", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.029684, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819558.8659022, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819563.909258, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819568.9596329, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819574.019552, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819581.222783, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819586.284042, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819591.3349, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819596.3755982, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819601.423213, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819606.467721, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819613.6467059, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819618.699002, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819623.755784, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819628.806924, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819633.865889, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819638.904032, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819646.123075, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819651.170353, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819656.2302139, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819661.2811131, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819666.3346581, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819671.37782, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819677.6328452, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819682.681368, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819687.726161, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819692.7660532, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819697.806262, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819702.854626, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819710.047392, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819715.0883682, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819720.151645, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819725.1986618, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819730.2535172, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819735.295785, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819742.488461, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819747.538282, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819752.6009831, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819757.649909, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819762.706199, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819767.74622, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819774.937818, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819779.981869, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819785.036659, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819790.081524, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819795.13225, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819800.1699588, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819806.6605718, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819811.713065, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819816.7576268, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819821.816402, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819826.855071, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819831.910625, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819838.716703, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819843.766697, + "phase": "db_recovery_1", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.058697, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819848.8267, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819853.876805, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819858.925164, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819863.966346, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819871.139872, + "phase": "db_recovery_1", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.045205, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819876.182864, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819881.221048, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819886.266845, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819891.329261, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819896.376587, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819903.570528, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819908.6251578, + "phase": "db_recovery_1", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819911.859421, + "phase": "worker_short_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819917.0881128, + "phase": "worker_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819924.295424, + "phase": "worker_short_recovery_2", + "alerts": [], + "pending": 3.0, + "db_lock_waiting": 0.0, + "queue_age": 4.549477, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819929.3570838, + "phase": "worker_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819934.4052758, + "phase": "worker_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819939.4480371, + "phase": "worker_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819944.501984, + "phase": "worker_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819949.5574908, + "phase": "worker_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819956.683379, + "phase": "worker_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819961.7500029, + "phase": "worker_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819966.796869, + "phase": "worker_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819971.842195, + "phase": "worker_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819976.883692, + "phase": "worker_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819977.235533, + "phase": "db_short_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819982.3795571, + "phase": "db_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819989.563594, + "phase": "db_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794819994.637486, + "phase": "db_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 4.488330341113106, + "ready_burn": 0.0 + }, + { + "at": 1794819999.699598, + "phase": "db_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.354058721934369, + "ready_burn": 0.0 + }, + { + "at": 1794820004.743309, + "phase": "db_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.37261698440208, + "ready_burn": 0.0 + }, + { + "at": 1794820009.796797, + "phase": "db_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.37261698440208, + "ready_burn": 0.0 + }, + { + "at": 1794820014.846712, + "phase": "db_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.37261698440208, + "ready_burn": 0.0 + }, + { + "at": 1794820022.02723, + "phase": "db_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.37261698440208, + "ready_burn": 0.0 + }, + { + "at": 1794820027.078769, + "phase": "db_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.372616984402079, + "ready_burn": 0.0 + }, + { + "at": 1794820032.133355, + "phase": "db_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.37261698440208, + "ready_burn": 0.0 + }, + { + "at": 1794820037.183817, + "phase": "db_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.37261698440208, + "ready_burn": 0.0 + }, + { + "at": 1794820042.223642, + "phase": "db_short_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.37261698440208, + "ready_burn": 0.0 + }, + { + "at": 1794820042.717522, + "phase": "worker_long_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.37261698440208, + "ready_burn": 0.0 + }, + { + "at": 1794820049.8869388, + "phase": "worker_long_2", + "alerts": [], + "pending": 2.0, + "db_lock_waiting": 0.0, + "queue_age": 2.788659, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820054.933075, + "phase": "worker_long_2", + "alerts": [], + "pending": 4.0, + "db_lock_waiting": 0.0, + "queue_age": 6.956118, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820059.980216, + "phase": "worker_long_2", + "alerts": [], + "pending": 7.0, + "db_lock_waiting": 0.0, + "queue_age": 12.099677, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820065.044092, + "phase": "worker_long_2", + "alerts": [], + "pending": 9.0, + "db_lock_waiting": 0.0, + "queue_age": 17.299884, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820070.100117, + "phase": "worker_long_2", + "alerts": [], + "pending": 12.0, + "db_lock_waiting": 0.0, + "queue_age": 22.435417, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820075.144098, + "phase": "worker_long_2", + "alerts": [], + "pending": 14.0, + "db_lock_waiting": 0.0, + "queue_age": 27.634681, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820082.335217, + "phase": "worker_long_2", + "alerts": [], + "pending": 16.0, + "db_lock_waiting": 0.0, + "queue_age": 31.834163, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820087.378607, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "3.1834163e+01" + } + ], + "pending": 19.0, + "db_lock_waiting": 0.0, + "queue_age": 36.993288, + "api_burn": 0.0, + "ready_burn": 8.333333333333332 + }, + { + "at": 1794820092.418299, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "3.6993288e+01" + } + ], + "pending": 22.0, + "db_lock_waiting": 0.0, + "queue_age": 42.214016, + "api_burn": 0.0, + "ready_burn": 33.33333333333333 + }, + { + "at": 1794820097.4632561, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "4.2214016e+01" + } + ], + "pending": 24.0, + "db_lock_waiting": 0.0, + "queue_age": 47.358252, + "api_burn": 0.0, + "ready_burn": 53.84615384615384 + }, + { + "at": 1794820102.527977, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "7.5e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "2.142857142857143e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "4.7358252e+01" + } + ], + "pending": 27.0, + "db_lock_waiting": 0.0, + "queue_age": 52.632258, + "api_burn": 0.0, + "ready_burn": 75.0 + }, + { + "at": 1794820107.571115, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "9.23076923076923e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "2.7906976744186046e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "5.2632258e+01" + } + ], + "pending": 29.0, + "db_lock_waiting": 0.0, + "queue_age": 57.782391, + "api_burn": 0.0, + "ready_burn": 92.3076923076923 + }, + { + "at": 1794820114.816596, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "3.80952380952381e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "6.1957976e+01" + } + ], + "pending": 34.0, + "db_lock_waiting": 0.0, + "queue_age": 67.099234, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820119.874914, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "4.5238095238095233e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "6.7099234e+01" + } + ], + "pending": 37.0, + "db_lock_waiting": 0.0, + "queue_age": 72.313206, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820124.935491, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "5.116279069767441e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "7.2313206e+01" + } + ], + "pending": 39.0, + "db_lock_waiting": 0.0, + "queue_age": 77.447334, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820129.987166, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "5.714285714285715e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "7.7447334e+01" + } + ], + "pending": 42.0, + "db_lock_waiting": 0.0, + "queue_age": 82.675101, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820135.040478, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "6.279069767441859e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "8.2675101e+01" + } + ], + "pending": 44.0, + "db_lock_waiting": 0.0, + "queue_age": 86.853675, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820140.0981948, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "6.744186046511628e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "8.6853675e+01" + } + ], + "pending": 46.0, + "db_lock_waiting": 0.0, + "queue_age": 91.995144, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820146.712168, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "7.38095238095238e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "9.1995144e+01" + } + ], + "pending": 49.0, + "db_lock_waiting": 0.0, + "queue_age": 97.196577, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820151.775434, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "8.095238095238095e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "9.7196577e+01" + } + ], + "pending": 52.0, + "db_lock_waiting": 0.0, + "queue_age": 102.356386, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820156.819336, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "8.604651162790697e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.02356386e+02" + } + ], + "pending": 54.0, + "db_lock_waiting": 0.0, + "queue_age": 107.641945, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820161.861467, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "9.285714285714286e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.07641945e+02" + } + ], + "pending": 57.0, + "db_lock_waiting": 0.0, + "queue_age": 112.799066, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820166.8972428, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "9.767441860465117e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.12799066e+02" + } + ], + "pending": 59.0, + "db_lock_waiting": 0.0, + "queue_age": 116.963861, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820171.9578161, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.16963861e+02" + } + ], + "pending": 62.0, + "db_lock_waiting": 0.0, + "queue_age": 122.121556, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820178.778079, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.22121556e+02" + } + ], + "pending": 64.0, + "db_lock_waiting": 0.0, + "queue_age": 127.333325, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820183.840703, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.27333325e+02" + } + ], + "pending": 67.0, + "db_lock_waiting": 0.0, + "queue_age": 132.526633, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820188.8870778, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.32526633e+02" + } + ], + "pending": 69.0, + "db_lock_waiting": 0.0, + "queue_age": 137.753032, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820193.931285, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.37753032e+02" + } + ], + "pending": 71.0, + "db_lock_waiting": 0.0, + "queue_age": 141.909463, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820198.9807591, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.41909463e+02" + } + ], + "pending": 74.0, + "db_lock_waiting": 0.0, + "queue_age": 147.094471, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820204.038552, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.47094471e+02" + } + ], + "pending": 77.0, + "db_lock_waiting": 0.0, + "queue_age": 152.237391, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820211.2117891, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.57363158e+02" + } + ], + "pending": 82.0, + "db_lock_waiting": 0.0, + "queue_age": 162.580248, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820216.263077, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.62580248e+02" + } + ], + "pending": 84.0, + "db_lock_waiting": 0.0, + "queue_age": 167.711614, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820221.331776, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.67711614e+02" + } + ], + "pending": 86.0, + "db_lock_waiting": 0.0, + "queue_age": 171.815583, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820226.3791401, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.71815583e+02" + } + ], + "pending": 89.0, + "db_lock_waiting": 0.0, + "queue_age": 176.965373, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820231.4232519, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.76965373e+02" + } + ], + "pending": 92.0, + "db_lock_waiting": 0.0, + "queue_age": 182.148936, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820236.4974039, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.82148936e+02" + } + ], + "pending": 94.0, + "db_lock_waiting": 0.0, + "queue_age": 187.345382, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820243.701325, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.87345382e+02" + } + ], + "pending": 97.0, + "db_lock_waiting": 0.0, + "queue_age": 192.49377, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820248.759543, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.9249377e+02" + } + ], + "pending": 99.0, + "db_lock_waiting": 0.0, + "queue_age": 197.635404, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820253.823819, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "1.97635404e+02" + } + ], + "pending": 101.0, + "db_lock_waiting": 0.0, + "queue_age": 201.828844, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820258.878646, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "2.01828844e+02" + } + ], + "pending": 104.0, + "db_lock_waiting": 0.0, + "queue_age": 207.018052, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820263.922422, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "2.07018052e+02" + } + ], + "pending": 107.0, + "db_lock_waiting": 0.0, + "queue_age": 212.228122, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820268.97691, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "2.12228122e+02" + } + ], + "pending": 109.0, + "db_lock_waiting": 0.0, + "queue_age": 217.372032, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820276.207819, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "2.22593931e+02" + } + ], + "pending": 114.0, + "db_lock_waiting": 0.0, + "queue_age": 227.742107, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820281.251297, + "phase": "worker_long_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "2.27742107e+02" + } + ], + "pending": 116.0, + "db_lock_waiting": 0.0, + "queue_age": 231.930007, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820283.00604, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "2.27742107e+02" + } + ], + "pending": 116.0, + "db_lock_waiting": 0.0, + "queue_age": 231.930007, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820290.197048, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "2.37136663e+02" + } + ], + "pending": 21.0, + "db_lock_waiting": 0.0, + "queue_age": 40.262939, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820295.247827, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:07:59.758725496Z", + "value": "4.0262939e+01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794820300.2994401, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "5.952380952380952e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "7.63888888888889e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 59.523809523809526 + }, + { + "at": 1794820305.364564, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "5.348837209302326e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "7.26027397260274e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 53.48837209302326 + }, + { + "at": 1794820310.40502, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "4.8837209302325585e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "6.986301369863014e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 48.83720930232558 + }, + { + "at": 1794820315.457363, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "4.285714285714286e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "6.666666666666666e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 42.85714285714286 + }, + { + "at": 1794820322.626465, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "3.571428571428571e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "6.25e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 35.71428571428571 + }, + { + "at": 1794820327.7076328, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "3.023255813953489e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "5.89041095890411e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 30.232558139534888 + }, + { + "at": 1794820332.767369, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "2.3809523809523808e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "5.555555555555556e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 23.809523809523807 + }, + { + "at": 1794820337.816441, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1.8604651162790695e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "5.205479452054795e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 18.604651162790695 + }, + { + "at": 1794820342.878847, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1.3953488372093026e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "4.931506849315069e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0299, + "api_burn": 0.0, + "ready_burn": 13.953488372093025 + }, + { + "at": 1794820347.935922, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "7.142857142857142e-02" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "4.5833333333333337e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 7.142857142857142 + }, + { + "at": 1794820355.159856, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "3.835616438356164e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820360.224361, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "3.472222222222222e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820365.2666209, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "3.150684931506849e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820370.311193, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "2.876712328767123e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820375.363198, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "2.4657534246575344e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820380.423444, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "2.0833333333333331e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820386.7661052, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1.780821917808219e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820391.8268921, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1.3888888888888887e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820396.8891442, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "1.1111111111111112e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820401.953394, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "8.21917808219178e-02" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820407.0026371, + "phase": "worker_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:08:14.758725496Z", + "value": "4.109589041095891e-02" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820412.051337, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820418.820318, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820423.880886, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.030037, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820428.9232519, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820433.976445, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820439.022662, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820444.0768058, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820451.248574, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820456.30625, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820461.3529532, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820466.393259, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820471.43727, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820476.4897852, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820483.698407, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820488.763179, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820493.822407, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820498.8695822, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820503.919018, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820508.9654772, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820516.200973, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820521.247678, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820526.292943, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820531.348215, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820536.404762, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820541.4621482, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820547.819256, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820552.878401, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820557.925208, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820562.973079, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820568.0171452, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820573.0664608, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820580.2564769, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820585.303862, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820590.354872, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820595.413693, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820600.476712, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820605.527362, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820612.745339, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.056212, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820617.791778, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820622.8358881, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820627.890975, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820632.933324, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820638.007235, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820645.197035, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820650.259012, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820655.303972, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820660.3457549, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820665.418894, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820670.474207, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820676.811253, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820681.8537002, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820686.905427, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820691.96495, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820697.000824, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820702.039055, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820708.83834, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820713.905123, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820718.950114, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820724.0229461, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.071574, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820729.073405, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820734.1348732, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820741.336373, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820746.386704, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820751.4350002, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820756.493618, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820761.534569, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820766.586145, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820773.771117, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820778.8170109, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820783.867597, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820788.923261, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820794.003569, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820799.06407, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820806.294699, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820811.349192, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820816.395524, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820821.445226, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820826.506969, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820831.565323, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820837.864084, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820842.906944, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820847.9771788, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820853.0231931, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820858.085063, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820863.128662, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820870.326598, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820875.3703601, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820880.421545, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820885.479959, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820890.5428789, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820895.60264, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820902.782113, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820907.840169, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820912.8821561, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820917.936843, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820922.989677, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820928.040203, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820935.256514, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820940.321461, + "phase": "worker_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820943.162766, + "phase": "db_long_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794820950.358294, + "phase": "db_long_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 2.862254025044723, + "ready_burn": 0.0 + }, + { + "at": 1794820955.4028711, + "phase": "db_long_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 11.8491921005386, + "ready_burn": 0.0 + }, + { + "at": 1794820960.454071, + "phase": "db_long_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 20.75134168157424, + "ready_burn": 0.0 + }, + { + "at": 1794820965.511235, + "phase": "db_long_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 29.80251346499102, + "ready_burn": 0.0 + }, + { + "at": 1794820970.576001, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "3.864042933810376e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "1.8166526492851134e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 38.64042933810376, + "ready_burn": 0.0 + }, + { + "at": 1794820975.63304, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "4.7755834829443444e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "2.239057239057239e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 47.755834829443444, + "ready_burn": 0.0 + }, + { + "at": 1794820982.813349, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "5.655296229802513e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "2.651515151515151e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 56.55296229802513, + "ready_burn": 0.0 + }, + { + "at": 1794820987.872853, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "6.570915619389587e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "3.0808080808080807e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 65.70915619389586, + "ready_burn": 0.0 + }, + { + "at": 1794820992.920567, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "7.450628366247757e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "3.493265993265994e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 74.50628366247757, + "ready_burn": 0.0 + }, + { + "at": 1794820997.963299, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "8.366247755834829e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "3.925863521482729e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 19.0, + "queue_age": 0.0, + "api_burn": 83.66247755834829, + "ready_burn": 0.0 + }, + { + "at": 1794821003.013938, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.245960502692998e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "4.335016835016835e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 92.45960502692998, + "ready_burn": 0.0 + }, + { + "at": 1794821008.076951, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "4.76832350463353e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794821015.2828648, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.532062391681108e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "5.61078348778433e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681107, + "ready_burn": 0.0 + }, + { + "at": 1794821020.327581, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "6.018518518518519e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794821025.3954492, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.531249999999999e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "6.450252951096122e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.31249999999999, + "ready_burn": 0.0 + }, + { + "at": 1794821030.4605792, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "6.860269360269361e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794821035.50655, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "7.293423271500844e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794821040.557645, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "7.708508845829823e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794821046.899721, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.532062391681108e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "8.136593591905564e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681107, + "ready_burn": 0.0 + }, + { + "at": 1794821051.949895, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.514731369150781e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "8.550968828980622e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 20.0, + "queue_age": 0.04657, + "api_burn": 95.1473136915078, + "ready_burn": 0.0 + }, + { + "at": 1794821056.999434, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "8.979763912310286e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794821062.046519, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.393428812131424e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794821067.088136, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.527363184079602e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794821072.152968, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.519469759734879e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794821078.951071, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.527363184079602e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794821084.0123289, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.519867549668873e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794821089.0602398, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.527363184079601e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794821094.102832, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794821099.150757, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.527754763877382e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794821104.1915488, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794821111.374979, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794821116.423397, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.527754763877382e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794821121.47722, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794821126.5422919, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.53287197231834e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.528145695364237e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3287197231834, + "ready_burn": 0.0 + }, + { + "at": 1794821131.601959, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794821136.65846, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794821143.866624, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.520264681555005e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794821148.9248588, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794821153.9678771, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.520264681555003e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794821159.0212522, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794821164.097211, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.520264681555005e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794821169.1558862, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.532062391681108e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681107, + "ready_burn": 0.0 + }, + { + "at": 1794821176.370825, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794821181.428454, + "phase": "db_long_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794821183.3303502, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794821190.516476, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "8.62646566164154e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.087947882736157e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 86.2646566164154, + "ready_burn": 0.0 + }, + { + "at": 1794821195.5625448, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "7.788944723618091e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "8.679706601466993e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 77.88944723618091, + "ready_burn": 0.0 + }, + { + "at": 1794821200.6064909, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "6.95142378559464e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "8.273615635179151e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 69.5142378559464, + "ready_burn": 0.0 + }, + { + "at": 1794821205.6604311, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "6.113902847571189e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "7.864710676446618e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 61.13902847571189, + "ready_burn": 0.0 + }, + { + "at": 1794821210.7265139, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "5.276381909547739e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "7.45928338762215e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 52.763819095477395, + "ready_burn": 0.0 + }, + { + "at": 1794821215.78102, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "4.438860971524289e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "7.049714751426243e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 44.388609715242886, + "ready_burn": 0.0 + }, + { + "at": 1794821222.97877, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "3.601340033500838e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "6.644951140065146e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 36.01340033500838, + "ready_burn": 0.0 + }, + { + "at": 1794821228.0395362, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "2.7684563758389263e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "6.239804241435563e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 27.684563758389263, + "ready_burn": 0.0 + }, + { + "at": 1794821233.098659, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "1.9262981574539365e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "5.830618892508144e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 19.262981574539364, + "ready_burn": 0.0 + }, + { + "at": 1794821238.1609972, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "1.1055276381909548e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "5.424143556280587e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 11.055276381909549, + "ready_burn": 0.0 + }, + { + "at": 1794821243.201827, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "5.012224938875306e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 2.5125628140703515, + "ready_burn": 0.0 + }, + { + "at": 1794821248.247288, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "4.608482871125611e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821255.4833379, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "3.792822185970637e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821260.5338922, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "3.382233088834556e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.023075, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821265.645705, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "2.977161500815661e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821270.692384, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "2.5651465798045603e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821275.756573, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "2.1615008156606852e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821280.8251572, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "1.750814332247557e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821287.94023, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "1.3458401305057097e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821292.98786, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "9.364820846905537e-02" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821298.0500438, + "phase": "db_recovery_2", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:22:49.758725496Z", + "value": "5.378973105134475e-02" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821303.0896661, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821308.142924, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821313.192609, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821320.4057589, + "phase": "db_recovery_2", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.027049, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821325.452194, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821330.5289621, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821335.587809, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821340.6318269, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821345.672498, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821352.838612, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821357.88254, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821362.928289, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821367.9823868, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821373.039351, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821378.0904698, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821385.309997, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821390.3583832, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821395.4056058, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821400.4636202, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821405.513783, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821410.581811, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821416.89998, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821421.9588008, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821427.010245, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821432.057587, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821437.11085, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821442.1773891, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821448.980494, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821454.050245, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821459.1031098, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821464.149257, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821469.191987, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821474.238494, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821481.417713, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821486.457516, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821491.503186, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821496.562465, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821501.622916, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821506.674398, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821513.875582, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821518.925895, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821523.9707499, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821529.023296, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821534.074947, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821539.127106, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821546.3606489, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821551.4177752, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821556.473984, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821561.51919, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821566.579071, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821571.644479, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821578.018537, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821583.067055, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821588.127682, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821593.186465, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821598.252713, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821603.306654, + "phase": "db_recovery_2", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.04806, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821610.494455, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821615.558969, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821620.5989602, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821625.645231, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821630.70659, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821635.771188, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821642.985468, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821648.043408, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821653.1000268, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821658.172095, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821663.223918, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821668.272096, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821675.481427, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821680.543327, + "phase": "db_recovery_2", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.06284, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821685.597024, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821690.649303, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821695.705308, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821700.7552462, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821707.033077, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821712.095092, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821717.148824, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821722.2212758, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821727.263654, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821732.3113759, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821739.007755, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821744.055392, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821749.101315, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821754.166987, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821759.235106, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821764.287068, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821771.503861, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821776.576156, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821781.64032, + "phase": "db_recovery_2", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.022157, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821786.688375, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821791.746171, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821796.8075788, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821804.0018098, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821809.060309, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821814.111806, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821819.1729212, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821824.218067, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821829.2702909, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821836.584875, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821841.6346061, + "phase": "db_recovery_2", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821843.6777458, + "phase": "worker_short_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821848.932553, + "phase": "worker_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821856.196396, + "phase": "worker_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821861.2599869, + "phase": "worker_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821866.321285, + "phase": "worker_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821871.369975, + "phase": "worker_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821876.4178061, + "phase": "worker_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821881.475302, + "phase": "worker_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821888.021657, + "phase": "worker_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821893.071341, + "phase": "worker_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821898.137003, + "phase": "worker_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821903.187715, + "phase": "worker_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821908.247284, + "phase": "worker_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821909.087786, + "phase": "db_short_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821914.255303, + "phase": "db_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821921.438155, + "phase": "db_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.181347150259068, + "ready_burn": 0.0 + }, + { + "at": 1794821926.507232, + "phase": "db_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.199306759098786, + "ready_burn": 0.0 + }, + { + "at": 1794821931.550737, + "phase": "db_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.181347150259067, + "ready_burn": 0.0 + }, + { + "at": 1794821936.605369, + "phase": "db_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.1993067590987865, + "ready_burn": 0.0 + }, + { + "at": 1794821941.659164, + "phase": "db_short_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.05972, + "api_burn": 5.181347150259067, + "ready_burn": 0.0 + }, + { + "at": 1794821946.705046, + "phase": "db_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.1993067590987865, + "ready_burn": 0.0 + }, + { + "at": 1794821953.867743, + "phase": "db_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.190311418685121, + "ready_burn": 0.0 + }, + { + "at": 1794821958.919838, + "phase": "db_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.1993067590987865, + "ready_burn": 0.0 + }, + { + "at": 1794821963.970783, + "phase": "db_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.1993067590987865, + "ready_burn": 0.0 + }, + { + "at": 1794821969.022142, + "phase": "db_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.1993067590987865, + "ready_burn": 0.0 + }, + { + "at": 1794821974.091979, + "phase": "db_short_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 3.8525963149078732, + "ready_burn": 0.0 + }, + { + "at": 1794821974.649665, + "phase": "worker_long_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 3.8525963149078732, + "ready_burn": 0.0 + }, + { + "at": 1794821981.850312, + "phase": "worker_long_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.563672, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821986.900141, + "phase": "worker_long_3", + "alerts": [], + "pending": 3.0, + "db_lock_waiting": 0.0, + "queue_age": 5.734793, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821991.958095, + "phase": "worker_long_3", + "alerts": [], + "pending": 5.0, + "db_lock_waiting": 0.0, + "queue_age": 9.85176, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794821997.009391, + "phase": "worker_long_3", + "alerts": [], + "pending": 8.0, + "db_lock_waiting": 0.0, + "queue_age": 15.044213, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822002.0636969, + "phase": "worker_long_3", + "alerts": [], + "pending": 11.0, + "db_lock_waiting": 0.0, + "queue_age": 20.186119, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822007.12391, + "phase": "worker_long_3", + "alerts": [], + "pending": 13.0, + "db_lock_waiting": 0.0, + "queue_age": 25.331777, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822014.30176, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "3.0621715e+01" + } + ], + "pending": 16.0, + "db_lock_waiting": 0.0, + "queue_age": 30.621715, + "api_burn": 0.0, + "ready_burn": 7.6923076923076925 + }, + { + "at": 1794822019.3496451, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "3.4763571e+01" + } + ], + "pending": 18.0, + "db_lock_waiting": 0.0, + "queue_age": 39.937108, + "api_burn": 0.0, + "ready_burn": 25.0 + }, + { + "at": 1794822024.411948, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "3.9937108e+01" + } + ], + "pending": 23.0, + "db_lock_waiting": 0.0, + "queue_age": 45.085746, + "api_burn": 0.0, + "ready_burn": 41.666666666666664 + }, + { + "at": 1794822029.458175, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "6.666666666666666e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "4.5085746e+01" + } + ], + "pending": 26.0, + "db_lock_waiting": 0.0, + "queue_age": 50.310686, + "api_burn": 0.0, + "ready_burn": 66.66666666666666 + }, + { + "at": 1794822034.521014, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "8.461538461538461e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "2.5581395348837205e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "5.0310686e+01" + } + ], + "pending": 28.0, + "db_lock_waiting": 0.0, + "queue_age": 55.43069, + "api_burn": 0.0, + "ready_burn": 84.61538461538461 + }, + { + "at": 1794822039.5664508, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "3.0952380952380953e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "5.543069e+01" + } + ], + "pending": 31.0, + "db_lock_waiting": 0.0, + "queue_age": 60.215234, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822046.8104892, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "4.285714285714286e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "6.5419617e+01" + } + ], + "pending": 33.0, + "db_lock_waiting": 0.0, + "queue_age": 65.419617, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822051.863591, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "4.8837209302325574e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "7.0566701e+01" + } + ], + "pending": 36.0, + "db_lock_waiting": 0.0, + "queue_age": 70.566701, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822056.928133, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "5.348837209302325e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "7.5702155e+01" + } + ], + "pending": 38.0, + "db_lock_waiting": 0.0, + "queue_age": 75.702155, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822061.972676, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "5.952380952380952e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "7.9886024e+01" + } + ], + "pending": 40.0, + "db_lock_waiting": 0.0, + "queue_age": 79.886024, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822067.030783, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "6.666666666666666e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "8.5017044e+01" + } + ], + "pending": 43.0, + "db_lock_waiting": 0.0, + "queue_age": 85.017044, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822072.088077, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "7.209302325581395e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "9.014771e+01" + } + ], + "pending": 46.0, + "db_lock_waiting": 0.0, + "queue_age": 90.14771, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822079.035081, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "7.857142857142856e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "9.5371921e+01" + } + ], + "pending": 48.0, + "db_lock_waiting": 0.0, + "queue_age": 95.371921, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822084.0922089, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "8.372093023255813e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.00518477e+02" + } + ], + "pending": 51.0, + "db_lock_waiting": 0.0, + "queue_age": 100.518477, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822089.152947, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "9.047619047619048e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.05723966e+02" + } + ], + "pending": 53.0, + "db_lock_waiting": 0.0, + "queue_age": 105.723966, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822094.209997, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "9.523809523809523e-01" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.09917312e+02" + } + ], + "pending": 55.0, + "db_lock_waiting": 0.0, + "queue_age": 109.917312, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822099.25932, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.15131163e+02" + } + ], + "pending": 58.0, + "db_lock_waiting": 0.0, + "queue_age": 115.131163, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822104.317303, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.20286333e+02" + } + ], + "pending": 61.0, + "db_lock_waiting": 0.0, + "queue_age": 120.286333, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822111.516449, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.30630237e+02" + } + ], + "pending": 66.0, + "db_lock_waiting": 0.0, + "queue_age": 130.630237, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822116.584248, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.34746034e+02" + } + ], + "pending": 68.0, + "db_lock_waiting": 0.0, + "queue_age": 134.746034, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822121.659651, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.39937576e+02" + } + ], + "pending": 70.0, + "db_lock_waiting": 0.0, + "queue_age": 139.937576, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822126.720025, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.45170715e+02" + } + ], + "pending": 73.0, + "db_lock_waiting": 0.0, + "queue_age": 145.170715, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822131.765887, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.50324903e+02" + } + ], + "pending": 76.0, + "db_lock_waiting": 0.0, + "queue_age": 150.324903, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822136.822687, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.55526366e+02" + } + ], + "pending": 78.0, + "db_lock_waiting": 0.0, + "queue_age": 155.526366, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822144.004744, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.60741869e+02" + } + ], + "pending": 81.0, + "db_lock_waiting": 0.0, + "queue_age": 160.741869, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822149.0797582, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.64934474e+02" + } + ], + "pending": 83.0, + "db_lock_waiting": 0.0, + "queue_age": 164.934474, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822154.137442, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.70075249e+02" + } + ], + "pending": 86.0, + "db_lock_waiting": 0.0, + "queue_age": 170.075249, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822159.1997051, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.75223552e+02" + } + ], + "pending": 88.0, + "db_lock_waiting": 0.0, + "queue_age": 175.223552, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822164.256215, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.80447776e+02" + } + ], + "pending": 91.0, + "db_lock_waiting": 0.0, + "queue_age": 180.447776, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822169.317166, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.85652658e+02" + } + ], + "pending": 93.0, + "db_lock_waiting": 0.0, + "queue_age": 185.652658, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822176.548369, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.95054452e+02" + } + ], + "pending": 98.0, + "db_lock_waiting": 0.0, + "queue_age": 195.054452, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822181.6110342, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "2.00267134e+02" + } + ], + "pending": 101.0, + "db_lock_waiting": 0.0, + "queue_age": 200.267134, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822186.663628, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "2.05402163e+02" + } + ], + "pending": 103.0, + "db_lock_waiting": 0.0, + "queue_age": 205.402163, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822191.713119, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "2.10649465e+02" + } + ], + "pending": 106.0, + "db_lock_waiting": 0.0, + "queue_age": 210.649465, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822196.757371, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "2.14843721e+02" + } + ], + "pending": 108.0, + "db_lock_waiting": 0.0, + "queue_age": 214.843721, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822201.885727, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "2.20030627e+02" + } + ], + "pending": 111.0, + "db_lock_waiting": 0.0, + "queue_age": 220.030627, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822209.0437498, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "2.25172727e+02" + } + ], + "pending": 113.0, + "db_lock_waiting": 0.0, + "queue_age": 225.172727, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822214.120394, + "phase": "worker_long_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "2.3032373e+02" + } + ], + "pending": 116.0, + "db_lock_waiting": 0.0, + "queue_age": 230.32373, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822214.892261, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "2.35525531e+02" + } + ], + "pending": 118.0, + "db_lock_waiting": 0.0, + "queue_age": 235.525531, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822222.100251, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1e+00" + }, + { + "labels": { + "alertname": "QueueLag", + "severity": "warning" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "Unfinished attachment age exceeds 30s" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:09.758725496Z", + "value": "1.38751077e+02" + } + ], + "pending": 70.0, + "db_lock_waiting": 0.0, + "queue_age": 138.751077, + "api_burn": 0.0, + "ready_burn": 100.0 + }, + { + "at": 1794822227.1567, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "6.27906976744186e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "7.777777777777778e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 62.7906976744186 + }, + { + "at": 1794822232.208034, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "5.714285714285714e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "7.5e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.062894, + "api_burn": 0.0, + "ready_burn": 57.14285714285714 + }, + { + "at": 1794822237.254048, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "5e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "7.123287671232876e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 50.0 + }, + { + "at": 1794822242.301607, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "4.418604651162791e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "6.712328767123288e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 44.18604651162791 + }, + { + "at": 1794822247.3613682, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "3.8095238095238093e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "6.388888888888888e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 38.095238095238095 + }, + { + "at": 1794822254.551766, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "3.255813953488372e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "6.027397260273973e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 32.558139534883715 + }, + { + "at": 1794822259.603819, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "2.619047619047619e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "5.694444444444445e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.002901, + "api_burn": 0.0, + "ready_burn": 26.19047619047619 + }, + { + "at": 1794822264.665913, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "2.142857142857143e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "5.416666666666667e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 21.42857142857143 + }, + { + "at": 1794822269.715411, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "1.4285714285714288e-01" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "5e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 14.285714285714288 + }, + { + "at": 1794822274.7605581, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateFast", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:24.758725496Z", + "value": "9.302325581395349e-02" + }, + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "4.657534246575343e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 2.380952380952381 + }, + { + "at": 1794822279.831406, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "4.0277777777777773e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822287.104585, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "3.6986301369863017e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822292.165309, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "3.287671232876712e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822297.216429, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "2.916666666666667e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822302.2824519, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "2.602739726027397e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822307.328106, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "2.222222222222222e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822312.376438, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1.9444444444444445e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.052481, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822319.064506, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1.5277777777777776e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822324.118548, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "1.2328767123287672e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822329.1822321, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "8.333333333333334e-02" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822334.235187, + "phase": "worker_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ReadyBurnRateSlow", + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "ready SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:40:29.758725496Z", + "value": "5.479452054794521e-02" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822339.300715, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822344.3514152, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822351.546798, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822356.5996192, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822361.6531682, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822366.720247, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822371.7693398, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822376.83294, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822384.002342, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822389.049303, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822394.1048691, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822399.155942, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822404.2099352, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822409.269195, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822416.515562, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822421.5738869, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822426.639362, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822431.688318, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822436.745323, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822441.7936919, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.02844, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822448.075794, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822453.125772, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822458.174474, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822463.234883, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822468.286473, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822473.332592, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822480.5413098, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822485.5934138, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822490.645658, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822495.709406, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822500.782361, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.003962, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822505.843452, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822513.016619, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822518.082098, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822523.1422012, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822528.1898618, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822533.237943, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822538.2960708, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822545.5382972, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822550.604863, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822555.667331, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822560.736855, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822565.7882729, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822570.83858, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822577.0744488, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822582.132475, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822587.194484, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822592.2509198, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822597.293664, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822602.35588, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822609.142443, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822614.204963, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822619.255313, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822624.309782, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822629.363373, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822634.424941, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822641.6098702, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822646.674624, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822651.744704, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822656.794732, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822661.847499, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822666.910811, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822674.119491, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.067832, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822679.187436, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822684.2467918, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822689.309078, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822694.358253, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822699.408881, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.071549, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822706.636075, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822711.684602, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822716.755046, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822721.8063931, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822726.854836, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822731.9090161, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.095499, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822738.1343338, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822743.1927931, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822748.247324, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822753.29631, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822758.37351, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822763.426618, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.029149, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822770.599573, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822775.66623, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822780.720912, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822785.782667, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822790.829561, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.051844, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822795.875221, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822803.081934, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822808.13799, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822813.200473, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822818.266502, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822823.3303812, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822828.4050581, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822834.589833, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822839.647351, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822844.705944, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822849.767988, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.039799, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822854.8320372, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822859.884919, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822867.157324, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822872.21538, + "phase": "worker_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822875.075948, + "phase": "db_long_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822882.276469, + "phase": "db_long_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 8.0, + "queue_age": 0.060649, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794822887.325949, + "phase": "db_long_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 8.438061041292638, + "ready_burn": 0.0 + }, + { + "at": 1794822892.3723662, + "phase": "db_long_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 17.352415026833633, + "ready_burn": 0.0 + }, + { + "at": 1794822897.415453, + "phase": "db_long_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 26.391382405745063, + "ready_burn": 0.0 + }, + { + "at": 1794822902.4918349, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "3.5241502683363146e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "1.6568544995794784e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 35.241502683363144, + "ready_burn": 0.0 + }, + { + "at": 1794822907.541219, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "4.434470377019749e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "2.079124579124579e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 44.34470377019749, + "ready_burn": 0.0 + }, + { + "at": 1794822914.742094, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "5.313059033989267e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "2.4978973927670312e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 62.298025134649905, + "ready_burn": 0.0 + }, + { + "at": 1794822919.816589, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "7.101967799642218e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "3.3389402859545836e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 71.01967799642217, + "ready_burn": 0.0 + }, + { + "at": 1794822924.865144, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "8.025134649910233e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "pending", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "3.7626262626262624e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 80.25134649910233, + "ready_burn": 0.0 + }, + { + "at": 1794822929.926443, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "8.88888888888889e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "4.175084175084175e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 88.8888888888889, + "ready_burn": 0.0 + }, + { + "at": 1794822934.982975, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.529616724738676e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "4.6043771043771037e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.29616724738676, + "ready_burn": 0.0 + }, + { + "at": 1794822940.0406048, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "5.016835016835017e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 20.0, + "queue_age": 0.058425, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794822947.177479, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "5.450716090985678e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794822952.230786, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.514731369150781e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "5.858585858585859e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.1473136915078, + "ready_burn": 0.0 + }, + { + "at": 1794822957.2861462, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "6.29317607413648e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794822962.360394, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.514731369150781e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "6.7003367003367e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 19.0, + "queue_age": 0.0, + "api_burn": 95.1473136915078, + "ready_burn": 0.0 + }, + { + "at": 1794822967.418842, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "7.135636057287279e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794822972.4635782, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "7.542087542087542e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 20.0, + "queue_age": 0.006195, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794822979.140764, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "7.978096040438081e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794822984.196285, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "8.383838383838382e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794822989.264122, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.532062391681108e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "8.819561551433388e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681107, + "ready_burn": 0.0 + }, + { + "at": 1794822994.323821, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.225589225589226e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 19.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794822999.380286, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.526184538653366e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794823004.434798, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.51473136915078e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.51946975973488e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.14731369150779, + "ready_burn": 0.0 + }, + { + "at": 1794823011.6406322, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.514731369150781e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.519469759734879e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.1473136915078, + "ready_burn": 0.0 + }, + { + "at": 1794823016.6923258, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.527363184079602e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794823021.7439191, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.514731369150781e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.519469759734879e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.1473136915078, + "ready_burn": 0.0 + }, + { + "at": 1794823026.801173, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.5273631840796e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794823031.873912, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.515570934256055e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.519867549668873e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.15570934256054, + "ready_burn": 0.0 + }, + { + "at": 1794823036.933111, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.527363184079604e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794823044.1560268, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794823049.2106612, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.532062391681108e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.527754763877381e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681107, + "ready_burn": 0.0 + }, + { + "at": 1794823054.273904, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794823059.332834, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.527754763877382e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794823064.389376, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794823069.441908, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.532062391681108e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.527754763877383e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681107, + "ready_burn": 0.0 + }, + { + "at": 1794823076.688499, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.527754763877382e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794823081.744183, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794823086.8003578, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.532062391681108e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.52814569536424e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681107, + "ready_burn": 0.0 + }, + { + "at": 1794823091.85978, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.519867549668874e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 19.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794823096.9131908, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.532062391681108e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.528145695364238e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681107, + "ready_burn": 0.0 + }, + { + "at": 1794823101.9625, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.520264681555004e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794823108.848524, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.53206239168111e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.528145695364237e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 19.0, + "queue_age": 0.0, + "api_burn": 95.3206239168111, + "ready_burn": 0.0 + }, + { + "at": 1794823113.915769, + "phase": "db_long_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.516407599309153e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.520264681555003e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.16407599309153, + "ready_burn": 0.0 + }, + { + "at": 1794823115.273949, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.532062391681109e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.528145695364237e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 20.0, + "queue_age": 0.0, + "api_burn": 95.32062391681109, + "ready_burn": 0.0 + }, + { + "at": 1794823122.498254, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "8.946488294314382e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "9.242671009771987e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 89.46488294314382, + "ready_burn": 0.0 + }, + { + "at": 1794823127.5495749, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "8.107202680067002e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "8.835504885993486e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 81.07202680067002, + "ready_burn": 0.0 + }, + { + "at": 1794823132.598665, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "7.274247491638796e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "8.428338762214983e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.093271, + "api_burn": 72.74247491638795, + "ready_burn": 0.0 + }, + { + "at": 1794823137.659934, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "6.432160804020101e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "8.021172638436481e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 64.321608040201, + "ready_burn": 0.0 + }, + { + "at": 1794823142.714678, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "5.59463986599665e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "7.614006514657979e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 55.9463986599665, + "ready_burn": 0.0 + }, + { + "at": 1794823147.7909691, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "4.757118927973199e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "7.204563977180114e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 47.57118927973199, + "ready_burn": 0.0 + }, + { + "at": 1794823155.003329, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "3.082077051926298e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "6.389568052159739e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 30.82077051926298, + "ready_burn": 0.0 + }, + { + "at": 1794823160.0691001, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "2.2445561139028475e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "5.985342019543974e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 22.445561139028474, + "ready_burn": 0.0 + }, + { + "at": 1794823165.127978, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateFast", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 6x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "1.4093959731543626e-01" + }, + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "5.579119086460033e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 14.093959731543626, + "ready_burn": 0.0 + }, + { + "at": 1794823170.1756742, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "5.171009771986971e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 5.6951423785594635, + "ready_burn": 0.0 + }, + { + "at": 1794823175.300823, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "4.763458401305057e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823180.4023418, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "4.3566775244299677e-01" + } + ], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.055387, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823187.342025, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "3.9477977161500816e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823192.410859, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "3.5423452768729635e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823197.480039, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "3.132137030995106e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823202.554781, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "2.719869706840391e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823207.625581, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "2.3164763458401308e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823212.742048, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "1.905537459283388e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823219.377044, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "1.500815660685155e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823224.440983, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "1.0912052117263846e-01" + } + ], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823229.508086, + "phase": "db_recovery_3", + "alerts": [ + { + "labels": { + "alertname": "ApiBurnRateSlow", + "severity": "warning", + "slo": "api" + }, + "annotations": { + "runbook": "infra/otel/RUNBOOK.md", + "summary": "api SLO budget burning above 3x" + }, + "state": "firing", + "activeAt": "2026-11-16T09:54:59.758725496Z", + "value": "6.851549755301793e-02" + } + ], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.074562, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823234.5681632, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823239.628609, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823244.7324748, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823252.057018, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823257.129337, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823262.191224, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823267.3149152, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823272.3947031, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823277.46627, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823285.080182, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823290.2117708, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823295.294507, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823300.42578, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823305.526304, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823310.581579, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823317.191917, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823322.2487528, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823327.3019269, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823332.363256, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823337.4236622, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823342.505858, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823349.29674, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823354.3616881, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823359.4096901, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823364.4689422, + "phase": "db_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.037494, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823369.583651, + "phase": "db_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.075258, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823374.6552541, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823381.9299169, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823386.984479, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823392.05472, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823397.111142, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823402.1695611, + "phase": "db_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.094563, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823407.221691, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823414.497633, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823419.560651, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823424.619174, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823429.702891, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823434.769404, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823439.8344262, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823447.167337, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823452.240034, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823457.3013241, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823462.366688, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823467.422091, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823472.475085, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823479.227592, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823484.284226, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823489.351033, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823494.6031518, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823499.696152, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823504.754764, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823512.029264, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823517.1123, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823522.1724029, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823527.223252, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823532.293366, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823537.38859, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823544.758925, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823549.840054, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823554.922576, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823559.981545, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823565.045023, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823570.1036339, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823577.225866, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823582.313283, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823587.3701131, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823592.419109, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823597.470679, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823602.524999, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823609.306765, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823614.36374, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823619.4462569, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823624.518494, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823629.575906, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823634.629889, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823641.875777, + "phase": "db_recovery_3", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.042398, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823646.92928, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823651.990935, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823657.050997, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823662.1025631, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823667.1494231, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823674.42894, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823679.5425978, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823684.5969799, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823689.670073, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823694.722522, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823699.7683089, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823707.051391, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823712.116836, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823717.179275, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823722.2265759, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823727.280256, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823732.3321338, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823739.239944, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823744.334883, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823749.395149, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823754.463246, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823759.518441, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823764.590152, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823771.866004, + "phase": "db_recovery_3", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823775.8158588, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823783.029254, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823788.094063, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823793.161534, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823798.211761, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823803.2705238, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823808.329289, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823815.5981271, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823820.658506, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.058662, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823825.7109618, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823830.772257, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823835.828005, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823840.929246, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823847.2312489, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823852.285919, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.038355, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823857.344527, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823862.411294, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823867.4780369, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823872.5372002, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823879.336014, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823884.392985, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823889.441725, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823894.7636302, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823899.8376162, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823904.94002, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823912.2489119, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823917.3052852, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823922.374122, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823927.424974, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823932.4936142, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823937.5493782, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823944.899374, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823949.960835, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823955.0414479, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823960.1023622, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823965.20343, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823970.266409, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.033769, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823977.3204212, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823982.377642, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823987.4546618, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823992.508449, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794823997.5775151, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824002.6357892, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824009.277745, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824014.3860831, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824019.4454331, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824024.5329769, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824029.6237, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 1.0, + "db_lock_waiting": 0.0, + "queue_age": 0.052079, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824034.694348, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824041.971515, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824047.06533, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824052.1166568, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824057.175104, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824062.22877, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824067.303245, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824074.68655, + "phase": "tempo_unavailable", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824076.171412, + "phase": "tempo_recovery", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824083.4044318, + "phase": "tempo_recovery", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824088.481048, + "phase": "tempo_recovery", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824093.54669, + "phase": "tempo_recovery", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824098.600568, + "phase": "tempo_recovery", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824103.66726, + "phase": "tempo_recovery", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824108.717226, + "phase": "tempo_recovery", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824116.007188, + "phase": "tempo_recovery", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824121.058976, + "phase": "tempo_recovery", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824126.1234791, + "phase": "tempo_recovery", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824131.17947, + "phase": "tempo_recovery", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824136.266609, + "phase": "final_drain", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824143.5558, + "phase": "final_drain", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824148.633846, + "phase": "final_drain", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824153.705992, + "phase": "final_drain", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824158.771051, + "phase": "final_drain", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + }, + { + "at": 1794824163.8431842, + "phase": "final_drain", + "alerts": [], + "pending": 0.0, + "db_lock_waiting": 0.0, + "queue_age": 0.0, + "api_burn": 0.0, + "ready_burn": 0.0 + } + ], + "resources": [ + { + "at": 1794817081.278859, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.64%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "40.84%", + "MemUsage": "209.1MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "22.9MB / 13.7MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.89%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "41.33%", + "MemUsage": "211.6MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "5.97MB / 3.52MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.75%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.51%", + "MemUsage": "12.17MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "1.15MB / 3.04MB", + "PIDs": "8" + }, + { + "BlockIO": "5.46MB / 0B", + "CPUPerc": "0.50%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.76%", + "MemUsage": "91.55MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "91.5MB / 37.2MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.52%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.31%", + "MemUsage": "13.45MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "264kB / 624kB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817113.2816682, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.89%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "43.66%", + "MemUsage": "223.6MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "24.8MB / 16.3MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.96%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "43.92%", + "MemUsage": "224.9MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "7.16MB / 5.37MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.92%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.32%", + "MemUsage": "11.93MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "1.32MB / 3.43MB", + "PIDs": "8" + }, + { + "BlockIO": "5.57MB / 0B", + "CPUPerc": "0.50%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.15%", + "MemUsage": "92.54MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "92.7MB / 37.7MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.37%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.36%", + "MemUsage": "13.93MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "2.23MB / 2.95MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817145.656804, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.88%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "47.70%", + "MemUsage": "244.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "27MB / 19.7MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.85%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "47.75%", + "MemUsage": "244.5MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "9.02MB / 8.32MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.72%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.34%", + "MemUsage": "11.95MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "1.48MB / 3.81MB", + "PIDs": "8" + }, + { + "BlockIO": "5.57MB / 0B", + "CPUPerc": "0.41%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.15%", + "MemUsage": "92.55MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "93.7MB / 38MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.10%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.37%", + "MemUsage": "14.03MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "4.19MB / 5.28MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817178.03187, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.15%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "51.94%", + "MemUsage": "265.9MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "29.4MB / 23.3MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.65%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "51.78%", + "MemUsage": "265.1MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "11MB / 11.5MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.82%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.27%", + "MemUsage": "11.86MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "1.65MB / 4.22MB", + "PIDs": "8" + }, + { + "BlockIO": "5.57MB / 0B", + "CPUPerc": "0.41%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.69%", + "MemUsage": "96.49MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "94.9MB / 38.4MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.90%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.38%", + "MemUsage": "14.09MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "6.26MB / 7.74MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817210.432042, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.69%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.64%", + "MemUsage": "28.86MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "31.2MB / 26.3MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.40%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "6.07%", + "MemUsage": "31.07MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "13.3MB / 15MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.72%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.49%", + "MemUsage": "12.14MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "1.82MB / 4.62MB", + "PIDs": "8" + }, + { + "BlockIO": "5.57MB / 0B", + "CPUPerc": "3.25%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.13%", + "MemUsage": "89.92MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "95.9MB / 38.8MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.17%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.38%", + "MemUsage": "14.16MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "8.23MB / 10.1MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817242.275549, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.19%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.38%", + "MemUsage": "27.53MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "33.3MB / 29.5MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.57%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "5.89%", + "MemUsage": "30.14MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "15.5MB / 18.4MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.84%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.74%", + "MemUsage": "12.46MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "1.99MB / 5.01MB", + "PIDs": "8" + }, + { + "BlockIO": "5.57MB / 0B", + "CPUPerc": "0.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.36%", + "MemUsage": "90.53MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "97.1MB / 39.2MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.34%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.42%", + "MemUsage": "14.55MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "10.2MB / 12.4MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817274.701441, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.84%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.57%", + "MemUsage": "28.53MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "35.2MB / 32.6MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.70%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "5.97%", + "MemUsage": "30.58MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "17.6MB / 21.7MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.87%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.53%", + "MemUsage": "12.2MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "2.16MB / 5.41MB", + "PIDs": "8" + }, + { + "BlockIO": "5.57MB / 0B", + "CPUPerc": "4.35%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.94%", + "MemUsage": "92.02MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "98.2MB / 39.6MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.86%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.40%", + "MemUsage": "14.36MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "12.1MB / 14.7MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817307.058057, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.71%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.68%", + "MemUsage": "29.06MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "37MB / 35.6MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.92%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "5.95%", + "MemUsage": "30.47MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "20.1MB / 25.5MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.66%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.54%", + "MemUsage": "12.21MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "2.33MB / 5.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.57MB / 0B", + "CPUPerc": "0.49%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.34%", + "MemUsage": "93.03MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "99.3MB / 39.9MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.40%", + "MemUsage": "14.39MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "14.1MB / 17MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817339.475033, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.87%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.56%", + "MemUsage": "28.45MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "38.9MB / 38.7MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.84%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "5.92%", + "MemUsage": "30.29MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "22.4MB / 29MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.77%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.46%", + "MemUsage": "12.11MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "2.5MB / 6.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.57MB / 0B", + "CPUPerc": "1.18%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.75%", + "MemUsage": "91.53MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "100MB / 40.3MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.92%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.42%", + "MemUsage": "14.5MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "16.1MB / 19.4MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817371.2586849, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.69%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.62%", + "MemUsage": "28.75MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "40.9MB / 42MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.67%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "5.96%", + "MemUsage": "30.49MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "24.6MB / 32.3MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.98%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.48%", + "MemUsage": "12.14MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "2.66MB / 6.59MB", + "PIDs": "8" + }, + { + "BlockIO": "5.57MB / 0B", + "CPUPerc": "0.40%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.76%", + "MemUsage": "91.54MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "101MB / 40.7MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.43%", + "MemUsage": "14.62MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "18MB / 21.7MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817403.27834, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.67%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.34%", + "MemUsage": "27.32MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "43.3MB / 45.6MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.52%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "5.97%", + "MemUsage": "30.56MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "26.4MB / 35.3MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.76%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.48%", + "MemUsage": "12.13MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "2.83MB / 6.98MB", + "PIDs": "8" + }, + { + "BlockIO": "5.69MB / 0B", + "CPUPerc": "0.35%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.99%", + "MemUsage": "92.14MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "103MB / 41.1MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.17%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.45%", + "MemUsage": "14.84MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "20MB / 24MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817435.691214, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.88%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.66%", + "MemUsage": "28.97MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "45.6MB / 49.1MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.81%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "6.03%", + "MemUsage": "30.9MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "28.2MB / 38.3MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.69%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.47%", + "MemUsage": "12.12MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "3MB / 7.38MB", + "PIDs": "8" + }, + { + "BlockIO": "5.69MB / 0B", + "CPUPerc": "0.39%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.61%", + "MemUsage": "93.71MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "104MB / 41.4MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.18%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.45%", + "MemUsage": "14.88MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "21.9MB / 26.3MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817468.069466, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.96%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.57%", + "MemUsage": "28.5MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "47.6MB / 52.3MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.79%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "5.92%", + "MemUsage": "30.31MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "30.5MB / 41.9MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.85%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.55%", + "MemUsage": "12.22MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "3.17MB / 7.78MB", + "PIDs": "8" + }, + { + "BlockIO": "5.69MB / 0B", + "CPUPerc": "0.39%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.60%", + "MemUsage": "93.71MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "105MB / 41.8MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.16%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.46%", + "MemUsage": "14.9MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "23.9MB / 28.7MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817500.465096, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.60%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.62%", + "MemUsage": "28.78MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "49.5MB / 55.3MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.21%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "6.23%", + "MemUsage": "31.89MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "32.8MB / 45.5MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.75%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.48%", + "MemUsage": "12.14MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "3.34MB / 8.18MB", + "PIDs": "8" + }, + { + "BlockIO": "5.69MB / 0B", + "CPUPerc": "0.99%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.91%", + "MemUsage": "91.93MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "106MB / 42.2MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.09%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.48%", + "MemUsage": "15.17MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "25.9MB / 31MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817532.301592, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.52%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.56%", + "MemUsage": "28.47MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "51.6MB / 58.7MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.67%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "5.98%", + "MemUsage": "30.63MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "34.9MB / 48.7MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.79%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.56%", + "MemUsage": "12.23MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "3.51MB / 8.57MB", + "PIDs": "8" + }, + { + "BlockIO": "5.69MB / 0B", + "CPUPerc": "0.58%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.76%", + "MemUsage": "94.11MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "107MB / 42.6MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.96%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.47%", + "MemUsage": "15.03MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "27.9MB / 33.4MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817564.710171, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.66%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.60%", + "MemUsage": "28.65MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "53.8MB / 62.1MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.90%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "6.16%", + "MemUsage": "31.53MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "36.9MB / 51.9MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.05%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.93%", + "MemUsage": "12.71MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "3.68MB / 8.96MB", + "PIDs": "8" + }, + { + "BlockIO": "5.69MB / 0B", + "CPUPerc": "1.61%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.44%", + "MemUsage": "93.29MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "108MB / 43MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.42%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.24MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "29.9MB / 35.8MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817597.0966542, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.20%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.67%", + "MemUsage": "29.02MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "56MB / 65.5MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.78%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "5.98%", + "MemUsage": "30.64MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "38.9MB / 55.1MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.83%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.57%", + "MemUsage": "12.25MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "3.84MB / 9.36MB", + "PIDs": "8" + }, + { + "BlockIO": "5.82MB / 0B", + "CPUPerc": "0.55%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.07%", + "MemUsage": "92.35MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "109MB / 43.3MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.19%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.48%", + "MemUsage": "15.11MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "31.9MB / 38.1MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817629.549962, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.89%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.60%", + "MemUsage": "28.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "57.8MB / 68.6MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.98%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "5.95%", + "MemUsage": "30.47MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "41.3MB / 58.6MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.79%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.57%", + "MemUsage": "12.25MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "4.01MB / 9.76MB", + "PIDs": "8" + }, + { + "BlockIO": "5.82MB / 0B", + "CPUPerc": "1.93%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.37%", + "MemUsage": "90.55MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "110MB / 43.7MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.08%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.21MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "33.8MB / 40.4MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817661.3341062, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.29%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.59%", + "MemUsage": "28.64MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "59.6MB / 71.6MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.91%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "6.08%", + "MemUsage": "31.14MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "43.6MB / 62.2MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.94%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.57%", + "MemUsage": "12.25MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "4.18MB / 10.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.82MB / 0B", + "CPUPerc": "0.68%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.37%", + "MemUsage": "90.56MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "111MB / 44.1MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.25%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.21MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "35.8MB / 42.7MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817693.3515542, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.76%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.61%", + "MemUsage": "28.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "61.5MB / 74.5MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.74%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "6.14%", + "MemUsage": "31.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "46MB / 65.9MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.62%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.58%", + "MemUsage": "12.26MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "4.35MB / 10.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.82MB / 0B", + "CPUPerc": "0.43%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.10%", + "MemUsage": "87.3MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "112MB / 44.4MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.18%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.25MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "37.7MB / 45.1MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817725.762059, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.66%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.34%", + "MemUsage": "27.34MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "63.8MB / 78MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.79%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "5.98%", + "MemUsage": "30.62MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "48MB / 69.1MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.68%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.59%", + "MemUsage": "12.27MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "4.51MB / 10.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.82MB / 0B", + "CPUPerc": "0.31%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.01%", + "MemUsage": "87.05MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "114MB / 44.8MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.64%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.53MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "39.7MB / 47.4MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817758.130684, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.37%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.62%", + "MemUsage": "28.79MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "65.5MB / 81MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.02%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "5.97%", + "MemUsage": "30.57MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "50.5MB / 72.8MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.06%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.99%", + "MemUsage": "12.79MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "4.69MB / 11.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.82MB / 0B", + "CPUPerc": "0.64%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.18%", + "MemUsage": "87.5MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "115MB / 45.2MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.25%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.55MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "41.6MB / 49.7MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817790.60036, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.65%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.64%", + "MemUsage": "28.9MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "67.9MB / 84.6MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.26%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "6.27%", + "MemUsage": "32.09MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "52.2MB / 75.7MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.75%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.69%", + "MemUsage": "12.41MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "4.86MB / 11.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.82MB / 0B", + "CPUPerc": "0.40%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.11%", + "MemUsage": "89.88MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "116MB / 45.6MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.88%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.50%", + "MemUsage": "15.4MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "43.7MB / 52.2MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817822.359453, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.94%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.33%", + "MemUsage": "27.31MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "70.4MB / 88.4MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.18%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "6.07%", + "MemUsage": "31.06MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "53.9MB / 78.6MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.88%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.60%", + "MemUsage": "12.29MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "5.02MB / 12.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.82MB / 0B", + "CPUPerc": "0.46%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.85%", + "MemUsage": "89.21MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "117MB / 45.9MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.96%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.45MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "45.7MB / 54.5MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817854.810224, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.27%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.62%", + "MemUsage": "28.79MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "72.3MB / 91.5MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.00%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "6.08%", + "MemUsage": "31.15MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "56.2MB / 82MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.79%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.69%", + "MemUsage": "12.41MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "5.19MB / 12.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.82MB / 0B", + "CPUPerc": "7.37%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.79%", + "MemUsage": "89.06MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "118MB / 46.3MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.57%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.65MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "47.6MB / 56.8MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817887.1911428, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.99%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.69%", + "MemUsage": "29.14MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "74.5MB / 94.8MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.97%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "6.00%", + "MemUsage": "30.73MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "58.2MB / 85.3MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.77%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.57%", + "MemUsage": "12.25MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "5.36MB / 12.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.82MB / 0B", + "CPUPerc": "0.35%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.36%", + "MemUsage": "87.97MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "119MB / 46.7MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.15%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.67MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "49.6MB / 59.1MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817919.642546, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.76%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.65%", + "MemUsage": "28.94MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "76.2MB / 97.7MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.85%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "6.03%", + "MemUsage": "30.89MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "60.7MB / 89.1MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.69%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "9.77%", + "MemUsage": "12.51MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "5.53MB / 13.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.82MB / 0B", + "CPUPerc": "1.17%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.12%", + "MemUsage": "87.36MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "120MB / 47.1MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.66%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.7MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "51.6MB / 61.5MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817951.42129, + "phase": "baseline", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.93%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.65%", + "MemUsage": "28.94MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "78MB / 101MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.91%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "6.29%", + "MemUsage": "32.18MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "63.1MB / 92.6MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.83%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.33%", + "MemUsage": "14.5MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "5.69MB / 13.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.82MB / 0B", + "CPUPerc": "0.52%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.87%", + "MemUsage": "89.26MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "121MB / 47.4MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.09%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.71MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "53.5MB / 63.8MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794817981.3579059, + "phase": "worker_short_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.81%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.68%", + "MemUsage": "29.07MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "79.7MB / 104MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.86%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "8.54%", + "MemUsage": "43.72MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "65.3MB / 95.9MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.76%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.33%", + "MemUsage": "14.5MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "5.85MB / 14.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.49%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.76%", + "MemUsage": "88.98MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "122MB / 47.8MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794817987.47078, + "phase": "worker_short_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.88%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.71%", + "MemUsage": "29.24MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "80MB / 104MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.66%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "11.10%", + "MemUsage": "56.82MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "65.7MB / 96.6MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.78%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.33%", + "MemUsage": "14.5MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "5.88MB / 14.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.60%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.76%", + "MemUsage": "88.99MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "123MB / 47.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.07%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.09%", + "MemUsage": "11.16MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "378kB / 438kB", + "PIDs": "8" + } + ] + }, + { + "at": 1794818019.889787, + "phase": "worker_short_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.87%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.36%", + "MemUsage": "27.47MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "82.1MB / 107MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.20%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "13.80%", + "MemUsage": "70.67MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "68MB / 100MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.87%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.43%", + "MemUsage": "14.63MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "6.05MB / 14.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.29%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.76%", + "MemUsage": "89MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "124MB / 48.2MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.37%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.14%", + "MemUsage": "11.68MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "2.34MB / 2.76MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794818047.5958881, + "phase": "db_short_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.82%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.45%", + "MemUsage": "27.89MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "83.6MB / 110MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.16%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "13.87%", + "MemUsage": "71.01MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "69.9MB / 103MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.76%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.44%", + "MemUsage": "14.64MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "6.19MB / 14.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.46%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.12%", + "MemUsage": "89.9MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "125MB / 48.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.08%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.15%", + "MemUsage": "11.79MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "4.05MB / 4.8MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794818052.4268138, + "phase": "db_short_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.53%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.70%", + "MemUsage": "29.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "84MB / 110MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.38%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "13.78%", + "MemUsage": "70.58MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "70.2MB / 103MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.87%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.43%", + "MemUsage": "14.63MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "6.22MB / 14.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.59%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.12%", + "MemUsage": "89.91MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "125MB / 48.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.02%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.36%", + "MemUsage": "13.92MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "4.42MB / 5.23MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794818084.8304892, + "phase": "db_short_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.33%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.42%", + "MemUsage": "27.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "86.2MB / 114MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.81%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "13.93%", + "MemUsage": "71.32MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "72.3MB / 107MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.75%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.37%", + "MemUsage": "14.55MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "6.39MB / 15.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.10%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.73%", + "MemUsage": "91.48MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "126MB / 49MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.22%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.35%", + "MemUsage": "13.87MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "6.37MB / 7.55MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794818112.3965719, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.94%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.36%", + "MemUsage": "27.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "88MB / 117MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.59%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.07%", + "MemUsage": "72.02MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "74.1MB / 110MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.94%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.46%", + "MemUsage": "14.67MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "6.54MB / 15.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.59%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.73%", + "MemUsage": "91.48MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "127MB / 49.3MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794818144.810463, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.78%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.40%", + "MemUsage": "27.64MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "90.3MB / 120MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.69%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.18%", + "MemUsage": "72.58MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "75.9MB / 113MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.12%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.47%", + "MemUsage": "14.68MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "6.72MB / 16.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.93%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.73%", + "MemUsage": "91.48MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "128MB / 49.7MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794818177.206787, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.72%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.55%", + "MemUsage": "28.41MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "92.5MB / 124MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.68%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.09%", + "MemUsage": "72.15MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "78MB / 116MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.99%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.44%", + "MemUsage": "14.64MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "6.91MB / 16.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.46%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.96%", + "MemUsage": "92.07MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "129MB / 50MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794818209.6506088, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.01%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "5.21%", + "MemUsage": "26.66MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "94.7MB / 127MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.99%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.02%", + "MemUsage": "71.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "79.9MB / 119MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.31%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.48%", + "MemUsage": "14.69MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "7.11MB / 16.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "3.36%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.00%", + "MemUsage": "87.05MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "130MB / 50.3MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794818241.4443278, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.13%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.29%", + "MemUsage": "62.93MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "96.7MB / 130MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.94%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.15%", + "MemUsage": "72.45MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "82.2MB / 122MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.90%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.46%", + "MemUsage": "14.67MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "7.3MB / 17.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.69%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.88%", + "MemUsage": "86.73MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "131MB / 50.7MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794818273.441746, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.74%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.55%", + "MemUsage": "64.23MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "98.5MB / 133MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.58%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.10%", + "MemUsage": "72.19MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "84.6MB / 126MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.74%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.57%", + "MemUsage": "14.81MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "7.5MB / 17.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.38%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.88%", + "MemUsage": "86.73MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "132MB / 51MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794818305.832563, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.62%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.62%", + "MemUsage": "64.61MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "100MB / 136MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.26%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.19%", + "MemUsage": "72.68MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "87MB / 130MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.02%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.48%", + "MemUsage": "14.69MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "7.69MB / 18MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.55%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.18%", + "MemUsage": "90.06MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "133MB / 51.4MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794818338.3175411, + "phase": "worker_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.01%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.59%", + "MemUsage": "64.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "102MB / 139MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.23%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.09%", + "MemUsage": "72.16MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "89.1MB / 133MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.78%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.48%", + "MemUsage": "14.7MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "7.89MB / 18.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.85%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.28%", + "MemUsage": "90.32MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "134MB / 51.7MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794818352.438957, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.25%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.57%", + "MemUsage": "64.38MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "104MB / 141MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.71%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.12%", + "MemUsage": "72.28MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "89.8MB / 134MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.69%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.48%", + "MemUsage": "14.69MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "7.98MB / 18.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.38%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.22%", + "MemUsage": "90.16MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "134MB / 51.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "99.70%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.41MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "5.51MB / 6.1MB", + "PIDs": "10" + } + ] + }, + { + "at": 1794818384.8432589, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.97%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.95%", + "MemUsage": "66.32MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "106MB / 145MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.94%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.15%", + "MemUsage": "72.43MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "91.4MB / 137MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.84%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.48%", + "MemUsage": "14.7MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "8.15MB / 19MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "7.14%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.31%", + "MemUsage": "90.38MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "136MB / 52.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.97%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.32%", + "MemUsage": "13.57MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "16.6MB / 19.2MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818417.2262552, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.00%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.88%", + "MemUsage": "65.93MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "108MB / 148MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.65%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.11%", + "MemUsage": "72.24MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "93.7MB / 141MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.60%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.48%", + "MemUsage": "14.7MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "8.32MB / 19.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.46%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.69%", + "MemUsage": "91.38MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "137MB / 52.8MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.09%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.48%", + "MemUsage": "15.16MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "18.5MB / 21.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818449.6652539, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.88%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.95%", + "MemUsage": "66.28MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "110MB / 152MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.01%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.22%", + "MemUsage": "72.82MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "95.6MB / 144MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.73%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.63%", + "MemUsage": "14.89MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "8.49MB / 19.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.31%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.64%", + "MemUsage": "91.25MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "138MB / 53.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.12%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.48%", + "MemUsage": "15.17MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "20.5MB / 23.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818481.463906, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.12%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.95%", + "MemUsage": "66.29MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "112MB / 155MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.11%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.12%", + "MemUsage": "72.29MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "98MB / 147MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.02%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.54%", + "MemUsage": "14.77MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "8.65MB / 20.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.83%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.47%", + "MemUsage": "90.81MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "139MB / 53.5MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.28%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.48%", + "MemUsage": "15.2MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "22.4MB / 26.2MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818513.476259, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.69%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.94%", + "MemUsage": "66.24MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "114MB / 158MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.84%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.25%", + "MemUsage": "72.95MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "100MB / 151MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.57%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.54%", + "MemUsage": "14.77MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "8.82MB / 20.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.38%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.21%", + "MemUsage": "90.12MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "140MB / 53.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.10%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.21MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "24.4MB / 28.5MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818545.9055302, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.95%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.95%", + "MemUsage": "66.31MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "116MB / 161MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.29%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.22%", + "MemUsage": "72.82MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "103MB / 154MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.14%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.63%", + "MemUsage": "14.89MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "8.99MB / 21MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.69%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.70%", + "MemUsage": "88.83MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "141MB / 54.2MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.38%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.22MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "26.5MB / 31MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818578.375475, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.19%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.00%", + "MemUsage": "66.58MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "118MB / 164MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.02%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.14%", + "MemUsage": "72.41MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "104MB / 157MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.93%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.84%", + "MemUsage": "15.16MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "9.16MB / 21.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.35%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.23%", + "MemUsage": "90.18MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "142MB / 54.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.76%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.28MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "28.4MB / 33.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818610.8461812, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.65%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.77%", + "MemUsage": "65.4MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "120MB / 168MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.84%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.12%", + "MemUsage": "72.29MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "106MB / 161MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.74%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.63%", + "MemUsage": "14.88MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "9.33MB / 21.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.37%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.09%", + "MemUsage": "92.39MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "143MB / 55MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.44%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.25MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "30.4MB / 35.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818642.490931, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.98%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.94%", + "MemUsage": "66.25MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "122MB / 171MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.38%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.15%", + "MemUsage": "72.45MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "109MB / 164MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.40%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.66%", + "MemUsage": "14.93MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "9.5MB / 22.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "3.39%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.45%", + "MemUsage": "93.31MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "145MB / 55.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.84%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.50%", + "MemUsage": "15.35MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "32.4MB / 38MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818674.932714, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.95%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.01%", + "MemUsage": "66.62MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "124MB / 174MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.42%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.20%", + "MemUsage": "72.71MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "111MB / 168MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.75%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.68%", + "MemUsage": "14.95MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "9.66MB / 22.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.06%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.15%", + "MemUsage": "89.98MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "146MB / 55.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.84%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.25MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "34.3MB / 40.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818707.3771079, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.79%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.98%", + "MemUsage": "66.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "126MB / 177MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.95%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.10%", + "MemUsage": "72.18MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "113MB / 171MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.91%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.68%", + "MemUsage": "14.95MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "9.83MB / 22.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.78%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.73%", + "MemUsage": "91.47MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "147MB / 56.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.17%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.25MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "36.3MB / 42.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818739.815776, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.30%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.64%", + "MemUsage": "64.71MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "129MB / 181MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.63%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.27%", + "MemUsage": "73.04MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "115MB / 174MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.82%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.64%", + "MemUsage": "14.9MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "10MB / 23.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.14%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.95%", + "MemUsage": "89.47MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "148MB / 56.5MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.19%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.27MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "38.3MB / 45MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818771.466392, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.67%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.02%", + "MemUsage": "66.66MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "131MB / 184MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.72%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.17%", + "MemUsage": "72.53MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "117MB / 177MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.55%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.64%", + "MemUsage": "14.89MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "10.2MB / 23.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.47%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.96%", + "MemUsage": "92.07MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "149MB / 56.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.13%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.27MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "40.2MB / 47.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818803.510402, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.14%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.95%", + "MemUsage": "66.32MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "133MB / 188MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.91%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.22%", + "MemUsage": "72.83MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "119MB / 180MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.76%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.69%", + "MemUsage": "14.96MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "10.3MB / 24.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.47%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.89%", + "MemUsage": "94.45MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "150MB / 57.2MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.10%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.27MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "42.1MB / 49.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818835.9078689, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.71%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.00%", + "MemUsage": "66.58MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "136MB / 191MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.44%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.12%", + "MemUsage": "72.29MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "121MB / 183MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.92%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.72%", + "MemUsage": "15MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "10.5MB / 24.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.54%", + "MemUsage": "93.54MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "151MB / 57.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.23%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.51MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "44.1MB / 51.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818868.31842, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.05%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.75%", + "MemUsage": "65.3MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "137MB / 194MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.73%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.19%", + "MemUsage": "72.66MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "123MB / 187MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.04%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.69%", + "MemUsage": "14.96MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "10.7MB / 24.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.54%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.13%", + "MemUsage": "89.93MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "152MB / 58MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.15%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.55MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "46.2MB / 54.4MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818900.7935832, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.73%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.01%", + "MemUsage": "66.59MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "139MB / 198MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.70%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.20%", + "MemUsage": "72.73MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "125MB / 190MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.85%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.71%", + "MemUsage": "14.99MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "10.8MB / 25.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.40%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.13%", + "MemUsage": "89.93MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "153MB / 58.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.11%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.55MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "48MB / 56.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818932.547147, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.30%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.66%", + "MemUsage": "64.82MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "141MB / 200MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.77%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.41%", + "MemUsage": "73.8MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "128MB / 194MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.83%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.72%", + "MemUsage": "15MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11MB / 25.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.59%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.25%", + "MemUsage": "90.24MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "155MB / 58.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.39%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.55MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "50.1MB / 59.1MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818964.968724, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.12%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.94%", + "MemUsage": "66.23MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "143MB / 204MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.84%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.45%", + "MemUsage": "74MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "130MB / 197MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.71%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.82%", + "MemUsage": "15.12MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.2MB / 26.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.24%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.49%", + "MemUsage": "93.41MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "156MB / 59.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.64%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.57MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "52.1MB / 61.4MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794818997.3721201, + "phase": "worker_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.31%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.01%", + "MemUsage": "66.62MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "146MB / 207MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.49%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.17%", + "MemUsage": "72.56MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "132MB / 201MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.75%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.71%", + "MemUsage": "14.98MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.3MB / 26.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.43%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.64%", + "MemUsage": "88.69MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "157MB / 59.5MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.12%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.56MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "54MB / 63.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819012.586006, + "phase": "db_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.52%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.74%", + "MemUsage": "65.23MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "146MB / 209MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.79%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.30%", + "MemUsage": "73.21MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "133MB / 202MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.01%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.71%", + "MemUsage": "14.98MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.4MB / 26.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.75%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.64%", + "MemUsage": "88.69MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "157MB / 59.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.75%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.57MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "55MB / 64.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819045.024032, + "phase": "db_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.29%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.33%", + "MemUsage": "68.23MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "148MB / 211MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.33%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.49%", + "MemUsage": "74.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "134MB / 204MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.69%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.72%", + "MemUsage": "15.01MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.6MB / 27.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.99%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.51%", + "MemUsage": "88.35MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "158MB / 60.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.02%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.58MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "57MB / 67.2MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819077.4660878, + "phase": "db_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.17%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.30%", + "MemUsage": "68.11MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "150MB / 213MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.47%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.39%", + "MemUsage": "73.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "135MB / 205MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.73%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.73%", + "MemUsage": "15.01MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.8MB / 27.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.75%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.38%", + "MemUsage": "88.01MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "160MB / 60.5MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.14%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.58MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "58.9MB / 69.5MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819109.917303, + "phase": "db_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.63%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.28%", + "MemUsage": "68.01MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "152MB / 216MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.84%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.39%", + "MemUsage": "73.68MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "137MB / 207MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.70%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.73%", + "MemUsage": "15.02MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "11.9MB / 27.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.29%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.98%", + "MemUsage": "87MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "161MB / 60.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.35%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.62MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "61MB / 72MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819141.543788, + "phase": "db_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.34%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.28%", + "MemUsage": "68MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "154MB / 218MB", + "PIDs": "8" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.88%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.66%", + "MemUsage": "75.07MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "138MB / 209MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.66%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.53%", + "MemUsage": "14.76MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "12.1MB / 28.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.39%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.98%", + "MemUsage": "87MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "162MB / 61.3MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.18%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.59MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "62.8MB / 74.2MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819173.5722911, + "phase": "db_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.05%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.18%", + "MemUsage": "67.5MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "155MB / 220MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.11%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.70%", + "MemUsage": "75.26MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "140MB / 211MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.72%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.54%", + "MemUsage": "14.77MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "12.3MB / 28.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.44%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.04%", + "MemUsage": "92.26MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "163MB / 61.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.11%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.6MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "64.8MB / 76.5MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819205.9902542, + "phase": "db_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.24%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.00%", + "MemUsage": "66.55MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "157MB / 222MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.01%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.51%", + "MemUsage": "74.3MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "141MB / 212MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.84%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.68%", + "MemUsage": "14.95MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "12.4MB / 29MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.17%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.34%", + "MemUsage": "93.04MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "165MB / 62.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.25%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.62MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "66.9MB / 79MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819238.4330602, + "phase": "db_long_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.69%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.32%", + "MemUsage": "68.19MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "159MB / 224MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.70%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.68%", + "MemUsage": "75.15MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "143MB / 215MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.75%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.67%", + "MemUsage": "14.94MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "12.6MB / 29.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.46%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.27%", + "MemUsage": "92.86MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "166MB / 62.5MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.08%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.62MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "68.7MB / 81.1MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819253.626983, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.72%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.35%", + "MemUsage": "68.34MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "159MB / 225MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.57%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.72%", + "MemUsage": "75.38MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "144MB / 216MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.61%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.88%", + "MemUsage": "15.21MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "12.7MB / 29.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.48%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.83%", + "MemUsage": "94.29MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "166MB / 62.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.08%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.72%", + "MemUsage": "17.62MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "69.7MB / 82.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819286.041928, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.06%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.30%", + "MemUsage": "68.12MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "162MB / 228MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.67%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.31%", + "MemUsage": "73.25MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "146MB / 219MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.06%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.73%", + "MemUsage": "15.02MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "12.9MB / 30MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.41%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.98%", + "MemUsage": "92.12MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "168MB / 63MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.52%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.72%", + "MemUsage": "17.64MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "71.8MB / 84.8MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819318.458293, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.79%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.36%", + "MemUsage": "68.39MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "164MB / 232MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.37%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.36%", + "MemUsage": "73.52MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "148MB / 222MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.61%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.74%", + "MemUsage": "15.02MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "13MB / 30.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.64%", + "MemUsage": "91.23MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "169MB / 63.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.39%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.72%", + "MemUsage": "17.65MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "73.7MB / 87.1MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819350.9175549, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.69%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.37%", + "MemUsage": "68.48MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "166MB / 235MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.59%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.39%", + "MemUsage": "73.68MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "150MB / 226MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.67%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.76%", + "MemUsage": "15.06MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "13.2MB / 30.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.36%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.98%", + "MemUsage": "92.11MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "170MB / 63.8MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.86%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.73%", + "MemUsage": "17.66MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "75.7MB / 89.4MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819382.6024299, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.28%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.36%", + "MemUsage": "68.39MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "168MB / 238MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.58%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.42%", + "MemUsage": "73.82MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "152MB / 229MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.69%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.74%", + "MemUsage": "15.03MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "13.4MB / 31.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.48%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.42%", + "MemUsage": "90.67MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "171MB / 64.2MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.96%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.74%", + "MemUsage": "17.84MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "77.7MB / 91.8MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819415.066349, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.63%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.33%", + "MemUsage": "68.26MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "170MB / 241MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.61%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.37%", + "MemUsage": "73.58MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "154MB / 233MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.81%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.58%", + "MemUsage": "14.83MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "13.5MB / 31.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.95%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.57%", + "MemUsage": "91.06MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "172MB / 64.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.13%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.74%", + "MemUsage": "17.84MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "79.6MB / 94.1MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819447.533206, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.65%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.38%", + "MemUsage": "68.5MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "172MB / 245MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.88%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.53%", + "MemUsage": "74.42MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "156MB / 236MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.86%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.59%", + "MemUsage": "14.83MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "13.7MB / 32MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.71%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.57%", + "MemUsage": "91.06MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "173MB / 64.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.07%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.74%", + "MemUsage": "17.85MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "81.6MB / 96.4MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819480.0093231, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.42%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.33%", + "MemUsage": "68.27MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "174MB / 248MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.05%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.55%", + "MemUsage": "74.51MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "159MB / 239MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.81%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.59%", + "MemUsage": "14.84MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "13.9MB / 32.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.66%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.70%", + "MemUsage": "91.4MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "174MB / 65.3MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.19%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.73%", + "MemUsage": "17.69MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "83.6MB / 98.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819511.597378, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.99%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.35%", + "MemUsage": "68.33MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "176MB / 251MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.07%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.61%", + "MemUsage": "74.79MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "160MB / 242MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.83%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.62%", + "MemUsage": "14.87MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "14MB / 32.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.68%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.65%", + "MemUsage": "93.82MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "175MB / 65.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.73%", + "MemUsage": "17.68MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "85.5MB / 101MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819543.628082, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.96%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.35%", + "MemUsage": "68.33MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "179MB / 255MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.67%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.48%", + "MemUsage": "74.14MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "162MB / 245MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.56%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.59%", + "MemUsage": "14.84MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "14.2MB / 33.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.40%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.14%", + "MemUsage": "95.09MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "176MB / 66MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.14%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.73%", + "MemUsage": "17.71MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "87.4MB / 103MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819576.19643, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.58%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.32%", + "MemUsage": "68.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "181MB / 259MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.69%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.68%", + "MemUsage": "75.17MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "164MB / 248MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.76%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.59%", + "MemUsage": "14.84MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "14.4MB / 33.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.91%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.70%", + "MemUsage": "93.96MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "178MB / 66.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.48%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.73%", + "MemUsage": "17.72MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "89.5MB / 106MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819608.635893, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.49%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.38%", + "MemUsage": "68.48MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "183MB / 262MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.71%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.73%", + "MemUsage": "75.4MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "166MB / 251MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.76%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.60%", + "MemUsage": "14.85MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "14.5MB / 34MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.46%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.56%", + "MemUsage": "93.6MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "179MB / 66.8MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.10%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.73%", + "MemUsage": "17.74MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "91.5MB / 108MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819641.0978649, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.59%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.37%", + "MemUsage": "68.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "185MB / 265MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.58%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.67%", + "MemUsage": "75.11MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "168MB / 255MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.81%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.62%", + "MemUsage": "14.88MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "14.7MB / 34.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.54%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.94%", + "MemUsage": "94.57MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "180MB / 67.2MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.01%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.73%", + "MemUsage": "17.74MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "93.4MB / 110MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819672.624649, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.37%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.35%", + "MemUsage": "68.33MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "188MB / 269MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.98%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.44%", + "MemUsage": "73.92MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "170MB / 258MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.70%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.59%", + "MemUsage": "14.84MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "14.9MB / 34.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.48%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.01%", + "MemUsage": "92.18MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "181MB / 67.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.37%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.73%", + "MemUsage": "17.75MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "95.4MB / 113MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819705.025663, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.61%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.35%", + "MemUsage": "68.37MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "189MB / 272MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.07%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.70%", + "MemUsage": "75.24MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "172MB / 262MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.77%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.65%", + "MemUsage": "14.91MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "15MB / 35.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "1.68%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.57%", + "MemUsage": "93.61MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "182MB / 67.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.73%", + "MemUsage": "17.76MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "97.4MB / 115MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819737.4726279, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.07%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.39%", + "MemUsage": "68.54MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "191MB / 275MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.45%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.67%", + "MemUsage": "75.13MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "175MB / 265MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.85%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.69%", + "MemUsage": "14.97MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "15.2MB / 35.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.58%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.12%", + "MemUsage": "82.23MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "183MB / 68.3MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.45%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.73%", + "MemUsage": "17.75MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "99.3MB / 117MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819769.91537, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.71%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.42%", + "MemUsage": "68.71MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "194MB / 278MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.34%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.44%", + "MemUsage": "73.94MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "177MB / 268MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.77%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "12.00%", + "MemUsage": "15.36MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "15.4MB / 35.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.92%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.20%", + "MemUsage": "82.42MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "184MB / 68.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.37%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.73%", + "MemUsage": "17.76MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "101MB / 120MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819801.644386, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.71%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.45%", + "MemUsage": "68.85MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "196MB / 282MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.80%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.50%", + "MemUsage": "74.23MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "179MB / 272MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.68%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.65%", + "MemUsage": "14.91MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "15.5MB / 36.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.43%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.20%", + "MemUsage": "82.42MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "185MB / 69.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.12%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.74%", + "MemUsage": "17.77MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "103MB / 122MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819833.680943, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.39%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.47%", + "MemUsage": "68.98MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "198MB / 285MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.95%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.73%", + "MemUsage": "75.39MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "181MB / 274MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.84%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.80%", + "MemUsage": "15.1MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "15.7MB / 36.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.48%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.18%", + "MemUsage": "82.38MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "186MB / 69.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.16%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.74%", + "MemUsage": "17.77MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "105MB / 124MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819866.1273339, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.56%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.41%", + "MemUsage": "68.67MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "200MB / 289MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.89%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.71%", + "MemUsage": "75.33MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "183MB / 278MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.98%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.95%", + "MemUsage": "15.29MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "15.9MB / 37.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.42%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.39%", + "MemUsage": "88.04MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "188MB / 69.8MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.57%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.75%", + "MemUsage": "17.89MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "107MB / 127MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819898.536671, + "phase": "db_recovery_1", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.17%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.39%", + "MemUsage": "68.58MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "202MB / 292MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.75%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.72%", + "MemUsage": "75.38MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "185MB / 281MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.98%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.95%", + "MemUsage": "15.3MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "16.1MB / 37.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.37%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.39%", + "MemUsage": "90.6MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "189MB / 70.2MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.37%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.74%", + "MemUsage": "17.83MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "109MB / 129MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794819913.82404, + "phase": "worker_short_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.79%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.47%", + "MemUsage": "68.95MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "203MB / 293MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.25%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.71%", + "MemUsage": "75.34MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "186MB / 283MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.77%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.79%", + "MemUsage": "15.09MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "16.1MB / 37.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.47%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.30%", + "MemUsage": "92.93MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "189MB / 70.4MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794819919.263246, + "phase": "worker_short_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.12%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.39%", + "MemUsage": "68.54MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "204MB / 294MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.16%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.49%", + "MemUsage": "74.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "186MB / 283MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.82%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.79%", + "MemUsage": "15.09MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "16.2MB / 37.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.89MB / 0B", + "CPUPerc": "0.86%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.38%", + "MemUsage": "93.14MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "189MB / 70.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.12%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.29%", + "MemUsage": "13.23MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "493kB / 567kB", + "PIDs": "8" + } + ] + }, + { + "at": 1794819951.6730418, + "phase": "worker_short_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.63%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.43%", + "MemUsage": "68.74MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "206MB / 298MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.80%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.67%", + "MemUsage": "75.12MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "188MB / 286MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.71%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.79%", + "MemUsage": "15.09MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "16.3MB / 38.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.45%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.51%", + "MemUsage": "93.46MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "190MB / 70.8MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.12%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.33%", + "MemUsage": "13.64MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "2.46MB / 2.9MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794819979.3855202, + "phase": "db_short_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.88%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.44%", + "MemUsage": "68.8MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "208MB / 301MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.73%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.54%", + "MemUsage": "74.45MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "189MB / 289MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.77%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.82%", + "MemUsage": "15.13MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "16.5MB / 38.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.00%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.35%", + "MemUsage": "95.61MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "191MB / 71.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.09%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.34%", + "MemUsage": "13.71MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "4.17MB / 4.94MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794819984.52813, + "phase": "db_short_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.96%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.44%", + "MemUsage": "68.82MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "209MB / 301MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.39%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.37%", + "MemUsage": "73.58MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "190MB / 289MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.80%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.81%", + "MemUsage": "15.11MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "16.5MB / 38.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.44%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.35%", + "MemUsage": "95.61MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "192MB / 71.2MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.97%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.34%", + "MemUsage": "13.73MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "4.54MB / 5.37MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794820017.0104132, + "phase": "db_short_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.69%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.13%", + "MemUsage": "67.25MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "211MB / 305MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.84%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.35%", + "MemUsage": "73.49MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "192MB / 293MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.79%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.81%", + "MemUsage": "15.12MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "16.7MB / 39MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.87%", + "MemUsage": "91.82MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "193MB / 71.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.95%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.35%", + "MemUsage": "13.82MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "6.49MB / 7.68MB", + "PIDs": "8" + } + ] + }, + { + "at": 1794820044.861008, + "phase": "worker_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.96%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.10%", + "MemUsage": "67.05MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "213MB / 308MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.86%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.53%", + "MemUsage": "74.42MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "194MB / 295MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.80%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.81%", + "MemUsage": "15.11MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "16.8MB / 39.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.84%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.88%", + "MemUsage": "94.41MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "194MB / 71.9MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794820077.31524, + "phase": "worker_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.84%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.40%", + "MemUsage": "68.61MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "214MB / 311MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.70%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.41%", + "MemUsage": "73.78MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "196MB / 299MB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.06%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.61%", + "MemUsage": "14.87MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "17MB / 39.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.91%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.56%", + "MemUsage": "93.59MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "195MB / 72.3MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794820109.790627, + "phase": "worker_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.68%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.21%", + "MemUsage": "67.64MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "216MB / 314MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.87%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.76%", + "MemUsage": "75.56MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "198MB / 303MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.71%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.70%", + "MemUsage": "14.98MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "17.2MB / 40.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.40%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.44%", + "MemUsage": "95.84MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "196MB / 72.6MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794820141.699175, + "phase": "worker_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.88%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.42%", + "MemUsage": "68.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "218MB / 317MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.93%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.72%", + "MemUsage": "75.37MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "201MB / 306MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.91%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.71%", + "MemUsage": "14.99MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "17.4MB / 40.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.83%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.11%", + "MemUsage": "94.99MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "197MB / 72.9MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794820173.745158, + "phase": "worker_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.58%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.29%", + "MemUsage": "68.02MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "221MB / 320MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.05%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.89%", + "MemUsage": "76.25MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "202MB / 309MB", + "PIDs": "10" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.88%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.81%", + "MemUsage": "15.12MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "17.6MB / 40.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.46%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.91%", + "MemUsage": "94.5MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "198MB / 73.3MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794820206.1914291, + "phase": "worker_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.69%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.43%", + "MemUsage": "68.75MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "223MB / 324MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.09%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.50%", + "MemUsage": "74.25MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "204MB / 312MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.73%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.67%", + "MemUsage": "14.93MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "17.8MB / 41.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.77%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.12%", + "MemUsage": "95.02MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "199MB / 73.6MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794820238.665272, + "phase": "worker_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.86%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.20%", + "MemUsage": "67.59MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "225MB / 327MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.08%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.65%", + "MemUsage": "75MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "206MB / 315MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.87%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.67%", + "MemUsage": "14.94MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "18MB / 41.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.29%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.05%", + "MemUsage": "94.85MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "200MB / 73.9MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794820271.192487, + "phase": "worker_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.80%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.17%", + "MemUsage": "67.45MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "227MB / 331MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.43%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.76%", + "MemUsage": "75.55MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "208MB / 319MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.76%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.81%", + "MemUsage": "15.12MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "18.2MB / 42.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.43%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.99%", + "MemUsage": "94.69MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "201MB / 74.3MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794820285.161296, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.01%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.39%", + "MemUsage": "68.56MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "228MB / 332MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.54%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.48%", + "MemUsage": "74.14MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "209MB / 320MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.66%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.72%", + "MemUsage": "15.01MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "18.3MB / 42.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "6.96%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.09%", + "MemUsage": "94.96MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "201MB / 74.5MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "92.49%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.56%", + "MemUsage": "16.01MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "8.8MB / 9.9MB", + "PIDs": "11" + } + ] + }, + { + "at": 1794820317.6170292, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.09%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.44%", + "MemUsage": "68.8MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "230MB / 336MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.97%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.46%", + "MemUsage": "74.04MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "211MB / 323MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.03%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.72%", + "MemUsage": "15.01MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "18.4MB / 42.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.70%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.22%", + "MemUsage": "95.28MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "203MB / 75MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.13%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.28MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "16.6MB / 19.2MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820350.12637, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.47%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.26%", + "MemUsage": "67.91MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "232MB / 339MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.85%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.68%", + "MemUsage": "75.15MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "214MB / 327MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.85%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.73%", + "MemUsage": "15.02MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "18.6MB / 43MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.63%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.25%", + "MemUsage": "95.36MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "204MB / 75.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.39%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.42MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "18.7MB / 21.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820381.7543108, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.93%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.48%", + "MemUsage": "69.04MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "235MB / 342MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.76%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.72%", + "MemUsage": "75.35MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "216MB / 330MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.78%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.82%", + "MemUsage": "15.12MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "18.8MB / 43.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.73%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.64%", + "MemUsage": "96.37MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "205MB / 75.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.10%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.50%", + "MemUsage": "15.38MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "20.5MB / 23.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820413.785917, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.61%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.47%", + "MemUsage": "68.97MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "237MB / 346MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.21%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.73%", + "MemUsage": "75.43MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "218MB / 333MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.83%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.83%", + "MemUsage": "15.14MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "18.9MB / 43.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.60%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.39%", + "MemUsage": "95.72MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "206MB / 76.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.25%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.45MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "22.4MB / 26.2MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820446.236663, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.57%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.55%", + "MemUsage": "69.37MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "239MB / 349MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.64%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.44%", + "MemUsage": "73.96MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "220MB / 337MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.15%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.64%", + "MemUsage": "14.9MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "19.1MB / 44.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.39%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.88%", + "MemUsage": "91.86MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "207MB / 76.5MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.80%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.49MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "24.5MB / 28.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820478.6673741, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.81%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.47%", + "MemUsage": "68.98MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "241MB / 352MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.63%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.49%", + "MemUsage": "74.17MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "222MB / 340MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.80%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.83%", + "MemUsage": "15.14MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "19.3MB / 44.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.32%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.89%", + "MemUsage": "91.89MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "208MB / 76.8MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.16%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.49MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "26.4MB / 30.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820511.1815271, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.76%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.26%", + "MemUsage": "67.88MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "243MB / 355MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.12%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.72%", + "MemUsage": "75.35MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "224MB / 344MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.75%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.82%", + "MemUsage": "15.13MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "19.4MB / 45MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.96%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.95%", + "MemUsage": "94.59MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "209MB / 77.2MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.13%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.55%", + "MemUsage": "15.89MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "28.4MB / 33.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820542.7832818, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.46%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.52%", + "MemUsage": "69.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "245MB / 358MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.73%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.73%", + "MemUsage": "75.43MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "226MB / 347MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.85%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.72%", + "MemUsage": "15MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "19.6MB / 45.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.37%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.55%", + "MemUsage": "93.57MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "210MB / 77.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.24%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.56%", + "MemUsage": "15.99MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "30.4MB / 35.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820575.2434251, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.74%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.23%", + "MemUsage": "67.73MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "247MB / 362MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.82%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.77%", + "MemUsage": "75.6MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "229MB / 350MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.83%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.72%", + "MemUsage": "15MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "19.8MB / 45.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.05%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.11%", + "MemUsage": "97.57MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "212MB / 78MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.13%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.57%", + "MemUsage": "16.08MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "32.4MB / 38MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820607.713262, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.39%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.52%", + "MemUsage": "69.24MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "249MB / 365MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.95%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.58%", + "MemUsage": "74.63MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "231MB / 353MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.02%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.69%", + "MemUsage": "14.96MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "19.9MB / 46.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.64%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.03%", + "MemUsage": "97.36MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "213MB / 78.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.57%", + "MemUsage": "16.09MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "34.3MB / 40.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820640.177255, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.66%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.46%", + "MemUsage": "68.93MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "251MB / 368MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.94%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.73%", + "MemUsage": "75.41MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "233MB / 357MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.57%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.69%", + "MemUsage": "14.96MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "20.1MB / 46.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.75%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.87%", + "MemUsage": "96.95MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "214MB / 78.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.10%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.57%", + "MemUsage": "16.12MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "36.3MB / 42.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820671.777268, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.08%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.53%", + "MemUsage": "69.27MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "253MB / 371MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.92%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.49%", + "MemUsage": "74.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "235MB / 360MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.95%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.69%", + "MemUsage": "14.96MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "20.3MB / 47MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.67%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "39.16%", + "MemUsage": "100.2MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "215MB / 79.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.16%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.57%", + "MemUsage": "16.08MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "38.2MB / 45MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820703.8171809, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.27%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.58%", + "MemUsage": "69.52MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "255MB / 375MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.11%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.81%", + "MemUsage": "75.82MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "237MB / 363MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.77%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.69%", + "MemUsage": "14.96MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "20.4MB / 47.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.57%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.73%", + "MemUsage": "99.16MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "216MB / 79.5MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.21%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.56%", + "MemUsage": "15.94MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "40.2MB / 47.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820736.300363, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.29%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.27%", + "MemUsage": "67.95MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "257MB / 378MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.94%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.81%", + "MemUsage": "75.84MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "239MB / 367MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.79%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.83%", + "MemUsage": "15.14MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "20.6MB / 47.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.17%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.26%", + "MemUsage": "97.95MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "217MB / 79.8MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.13%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.58%", + "MemUsage": "16.21MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "42.3MB / 49.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820768.75071, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.88%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.50%", + "MemUsage": "69.11MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "260MB / 382MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.29%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.78%", + "MemUsage": "75.65MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "241MB / 369MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.89%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.83%", + "MemUsage": "15.14MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "20.8MB / 48.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.53%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.50%", + "MemUsage": "95.99MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "218MB / 80.2MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.85%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.56%", + "MemUsage": "16MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "44.2MB / 52.1MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820801.260994, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.25%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.29%", + "MemUsage": "68.05MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "262MB / 385MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.07%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.78%", + "MemUsage": "75.68MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "243MB / 373MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.04%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.74%", + "MemUsage": "15.03MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "21MB / 48.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.87%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.77%", + "MemUsage": "96.68MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "219MB / 80.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.17%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.56%", + "MemUsage": "16.02MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "46.2MB / 54.4MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820832.8496482, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.64%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.20%", + "MemUsage": "67.61MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "264MB / 388MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.55%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.77%", + "MemUsage": "75.6MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "245MB / 377MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.93%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.74%", + "MemUsage": "15.03MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "21.1MB / 48.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.31%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.59%", + "MemUsage": "93.68MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "220MB / 81MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.53%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.57%", + "MemUsage": "16.04MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "48.1MB / 56.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820865.2903922, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.89%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.47%", + "MemUsage": "68.95MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "265MB / 391MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.85%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.79%", + "MemUsage": "75.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "248MB / 380MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.89%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "12.03%", + "MemUsage": "15.39MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "21.3MB / 49.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "5.20%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "39.15%", + "MemUsage": "100.2MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "222MB / 81.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.15%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.59%", + "MemUsage": "16.24MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "50.1MB / 59.1MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820897.7600489, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.22%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.51%", + "MemUsage": "69.16MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "267MB / 394MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.09%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.90%", + "MemUsage": "76.29MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "250MB / 384MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.00%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.65%", + "MemUsage": "14.91MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "21.5MB / 49.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.88%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.82%", + "MemUsage": "96.83MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "223MB / 81.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.11%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.58%", + "MemUsage": "16.2MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "52.1MB / 61.4MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820930.2139368, + "phase": "worker_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.29%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.49%", + "MemUsage": "69.08MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "269MB / 398MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.22%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.76%", + "MemUsage": "75.57MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "252MB / 387MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.10%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.67%", + "MemUsage": "14.93MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "21.6MB / 50.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.54%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.70%", + "MemUsage": "93.95MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "224MB / 82.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.58%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.57%", + "MemUsage": "16.1MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "54.1MB / 63.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820945.3190222, + "phase": "db_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.62%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.50%", + "MemUsage": "69.12MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "270MB / 399MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.62%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.81%", + "MemUsage": "75.85MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "253MB / 388MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.87%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "12.03%", + "MemUsage": "15.4MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "21.7MB / 50.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.22%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.41%", + "MemUsage": "95.77MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "224MB / 82.3MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.26%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.57%", + "MemUsage": "16.11MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "55MB / 64.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794820977.796012, + "phase": "db_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.35%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.39%", + "MemUsage": "68.57MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "272MB / 401MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.97%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.85%", + "MemUsage": "76.05MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "255MB / 391MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.86%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.85%", + "MemUsage": "15.16MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "21.9MB / 50.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.38%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.99%", + "MemUsage": "99.82MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "226MB / 82.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.10%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.60%", + "MemUsage": "16.36MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "57MB / 67.2MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821010.2462451, + "phase": "db_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.87%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.19%", + "MemUsage": "67.51MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "274MB / 403MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.28%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.79%", + "MemUsage": "75.74MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "256MB / 392MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.95%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.61%", + "MemUsage": "14.86MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "22MB / 51.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.88%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.16%", + "MemUsage": "97.7MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "227MB / 83.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.10%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.60%", + "MemUsage": "16.41MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "58.9MB / 69.5MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821041.8881311, + "phase": "db_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.71%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.23%", + "MemUsage": "67.75MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "275MB / 405MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "3.43%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.93%", + "MemUsage": "76.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "258MB / 395MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.02%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.62%", + "MemUsage": "14.88MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "22.2MB / 51.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.79%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.49%", + "MemUsage": "93.43MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "228MB / 83.5MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.11%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.60%", + "MemUsage": "16.43MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "60.9MB / 71.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821073.9126499, + "phase": "db_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.61%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.52%", + "MemUsage": "69.21MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "276MB / 407MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.20%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.83%", + "MemUsage": "75.93MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "260MB / 397MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.88%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.61%", + "MemUsage": "14.86MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "22.4MB / 51.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.38%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.75%", + "MemUsage": "94.08MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "229MB / 83.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.28%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.60%", + "MemUsage": "16.39MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "62.9MB / 74.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821106.358484, + "phase": "db_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.62%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.67%", + "MemUsage": "70MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "278MB / 409MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.34%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.48%", + "MemUsage": "74.14MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "261MB / 399MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.83%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.75%", + "MemUsage": "15.04MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "22.5MB / 52.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.34%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.49%", + "MemUsage": "98.53MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "230MB / 84.3MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.13%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.58%", + "MemUsage": "16.23MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "64.8MB / 76.5MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821138.827126, + "phase": "db_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.37%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.55%", + "MemUsage": "69.36MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "279MB / 411MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.45%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.80%", + "MemUsage": "75.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "263MB / 401MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.75%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.75%", + "MemUsage": "15.04MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "22.7MB / 52.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.56%", + "MemUsage": "96.16MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "232MB / 84.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.87%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.61%", + "MemUsage": "16.45MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "66.9MB / 79MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821171.358825, + "phase": "db_long_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.51%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.62%", + "MemUsage": "69.73MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "281MB / 413MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.49%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.71%", + "MemUsage": "75.32MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "264MB / 402MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.74%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.70%", + "MemUsage": "14.98MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "22.9MB / 53.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.40%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.28%", + "MemUsage": "95.44MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "233MB / 85.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.06%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.59%", + "MemUsage": "16.3MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "68.8MB / 81.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821185.476694, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.19%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.52%", + "MemUsage": "69.22MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "282MB / 414MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.13%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.75%", + "MemUsage": "75.51MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "265MB / 403MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.71%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.71%", + "MemUsage": "14.98MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "23MB / 53.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.55%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.18%", + "MemUsage": "95.18MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "233MB / 85.3MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.12%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.58%", + "MemUsage": "16.21MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "69.7MB / 82.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821217.952563, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.73%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.28%", + "MemUsage": "67.97MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "284MB / 417MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.16%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.49%", + "MemUsage": "74.18MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "267MB / 407MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.73%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.71%", + "MemUsage": "14.99MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "23.1MB / 53.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.64%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.01%", + "MemUsage": "92.18MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "235MB / 85.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.18%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.58%", + "MemUsage": "16.23MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "71.8MB / 84.8MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821250.447724, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.93%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.52%", + "MemUsage": "69.22MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "286MB / 420MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.65%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.53%", + "MemUsage": "74.41MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "270MB / 411MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.93%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.81%", + "MemUsage": "15.12MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "23.3MB / 54MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.35%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.92%", + "MemUsage": "91.95MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "236MB / 86MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.90%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.61%", + "MemUsage": "16.51MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "73.7MB / 87.1MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821282.924855, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.16%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.53%", + "MemUsage": "69.27MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "288MB / 423MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.66%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.77%", + "MemUsage": "75.6MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "272MB / 414MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.76%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "12.25%", + "MemUsage": "15.68MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "23.5MB / 54.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.32%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.66%", + "MemUsage": "93.84MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "237MB / 86.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.75%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.62%", + "MemUsage": "16.56MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "75.7MB / 89.5MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821315.368216, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.51%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.58%", + "MemUsage": "69.55MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "290MB / 426MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.40%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.80%", + "MemUsage": "75.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "274MB / 418MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.60%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.71%", + "MemUsage": "14.98MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "23.6MB / 54.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "4.66%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.84%", + "MemUsage": "96.87MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "238MB / 86.8MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.08%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.62%", + "MemUsage": "16.59MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "77.6MB / 91.8MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821347.822835, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.97%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.61%", + "MemUsage": "69.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "292MB / 430MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.07%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.80%", + "MemUsage": "75.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "276MB / 421MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.88%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.72%", + "MemUsage": "15MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "23.8MB / 55.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.76%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.84%", + "MemUsage": "96.86MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "239MB / 87.2MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.41%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.87%", + "MemUsage": "19.2MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "79.7MB / 94.1MB", + "PIDs": "10" + } + ] + }, + { + "at": 1794821380.269038, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.00%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.63%", + "MemUsage": "69.79MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "294MB / 433MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.10%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.49%", + "MemUsage": "74.18MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "279MB / 424MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.95%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.52%", + "MemUsage": "14.75MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "24MB / 55.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.03%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.61%", + "MemUsage": "93.73MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "240MB / 87.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.57%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.62%", + "MemUsage": "16.6MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "81.7MB / 96.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821411.884062, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.80%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.45%", + "MemUsage": "68.85MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "296MB / 436MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.33%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.05%", + "MemUsage": "71.96MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "281MB / 428MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.74%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.53%", + "MemUsage": "14.75MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "24.1MB / 56MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.50%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.07%", + "MemUsage": "94.89MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "241MB / 87.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.82%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.64%", + "MemUsage": "16.8MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "83.6MB / 98.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821443.935189, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.30%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.66%", + "MemUsage": "69.93MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "298MB / 439MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.81%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.34%", + "MemUsage": "73.41MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "283MB / 431MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.82%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.53%", + "MemUsage": "14.76MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "24.3MB / 56.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.43%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.20%", + "MemUsage": "95.23MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "242MB / 88.3MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.13%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.61%", + "MemUsage": "16.45MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "85.6MB / 101MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821476.403015, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.84%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.57%", + "MemUsage": "69.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "300MB / 442MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.31%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.43%", + "MemUsage": "73.9MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "285MB / 435MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.86%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.52%", + "MemUsage": "14.75MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "24.5MB / 56.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.65%", + "MemUsage": "93.81MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "244MB / 88.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.25%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.61%", + "MemUsage": "16.46MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "87.6MB / 104MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821508.837621, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.73%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.59%", + "MemUsage": "69.57MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "302MB / 446MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.78%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.19%", + "MemUsage": "72.68MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "287MB / 438MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.75%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "12.13%", + "MemUsage": "15.53MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "24.6MB / 57.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.56%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.64%", + "MemUsage": "93.81MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "245MB / 89MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.10%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.62%", + "MemUsage": "16.63MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "89.4MB / 106MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821541.346692, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.68%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.56%", + "MemUsage": "69.41MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "304MB / 450MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.70%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.48%", + "MemUsage": "74.13MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "289MB / 441MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.72%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.56%", + "MemUsage": "14.8MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "24.8MB / 57.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.77%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.22%", + "MemUsage": "95.29MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "246MB / 89.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.10%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.63%", + "MemUsage": "16.67MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "91.5MB / 108MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821572.981697, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.04%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.63%", + "MemUsage": "69.8MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "307MB / 453MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.02%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.23%", + "MemUsage": "72.86MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "291MB / 444MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.77%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.56%", + "MemUsage": "14.8MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "25MB / 58MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.75%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.54%", + "MemUsage": "93.54MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "247MB / 89.8MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.26%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.63%", + "MemUsage": "16.71MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "93.4MB / 111MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821605.474826, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.43%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.59%", + "MemUsage": "69.57MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "309MB / 456MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.82%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.55%", + "MemUsage": "74.48MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "293MB / 447MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.71%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.56%", + "MemUsage": "14.8MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "25.1MB / 58.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.83%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.54%", + "MemUsage": "93.55MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "248MB / 90.2MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.22%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.63%", + "MemUsage": "16.73MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "95.4MB / 113MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821637.93954, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.27%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.56%", + "MemUsage": "69.45MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "311MB / 460MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.67%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.49%", + "MemUsage": "74.17MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "295MB / 451MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.77%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.57%", + "MemUsage": "14.8MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "25.3MB / 58.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.56%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.13%", + "MemUsage": "92.49MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "249MB / 90.5MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.19%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.64%", + "MemUsage": "16.75MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "97.3MB / 115MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821670.457514, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.90%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.83%", + "MemUsage": "65.68MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "313MB / 463MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.77%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.50%", + "MemUsage": "74.25MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "297MB / 454MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.10%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.56%", + "MemUsage": "14.8MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "25.5MB / 59.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.75%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.56%", + "MemUsage": "93.6MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "250MB / 90.9MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.00%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.64%", + "MemUsage": "16.76MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "99.4MB / 118MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821701.995209, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.93%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.87%", + "MemUsage": "65.91MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "315MB / 466MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.98%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.49%", + "MemUsage": "74.21MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "299MB / 457MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.73%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.57%", + "MemUsage": "14.8MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "25.7MB / 59.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.65%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.46%", + "MemUsage": "93.35MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "251MB / 91.3MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.12%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.65%", + "MemUsage": "16.9MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "101MB / 120MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821733.992838, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.30%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "12.80%", + "MemUsage": "65.52MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "317MB / 470MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.00%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.27%", + "MemUsage": "73.08MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "301MB / 460MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.80%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.58%", + "MemUsage": "14.82MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "25.8MB / 59.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.64%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.07%", + "MemUsage": "92.35MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "252MB / 91.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.36%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.66%", + "MemUsage": "17.04MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "103MB / 122MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821766.4600818, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.94%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.03%", + "MemUsage": "66.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "319MB / 473MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.95%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.48%", + "MemUsage": "74.15MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "303MB / 464MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.58%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.57%", + "MemUsage": "14.81MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "26MB / 60.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.60%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.07%", + "MemUsage": "92.35MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "253MB / 92MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.86%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.64%", + "MemUsage": "16.79MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "105MB / 125MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821798.979352, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.64%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.10%", + "MemUsage": "67.09MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "322MB / 476MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.70%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.50%", + "MemUsage": "74.26MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "305MB / 467MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.85%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.57%", + "MemUsage": "14.81MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "26.2MB / 60.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.40%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.32%", + "MemUsage": "92.98MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "255MB / 92.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.25%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.62%", + "MemUsage": "16.62MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "107MB / 127MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821831.545897, + "phase": "db_recovery_2", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.01%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.12%", + "MemUsage": "67.18MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "324MB / 480MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.16%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.54%", + "MemUsage": "74.44MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "307MB / 470MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.71%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.65%", + "MemUsage": "14.91MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "26.3MB / 61.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "2.15%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.13%", + "MemUsage": "97.62MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "256MB / 92.8MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.36%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.64%", + "MemUsage": "16.79MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "109MB / 129MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821845.832313, + "phase": "worker_short_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.85%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.12%", + "MemUsage": "67.18MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "324MB / 481MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.57%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.53%", + "MemUsage": "74.38MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "309MB / 472MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.01%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.68%", + "MemUsage": "14.95MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "26.4MB / 61.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.55%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.34%", + "MemUsage": "98.16MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "256MB / 93MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794821851.149737, + "phase": "worker_short_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.44%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.05%", + "MemUsage": "66.83MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "325MB / 482MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.77%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.52%", + "MemUsage": "74.36MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "309MB / 472MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.92%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.67%", + "MemUsage": "14.93MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "26.4MB / 61.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.44%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.35%", + "MemUsage": "98.18MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "256MB / 93MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.08%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.10%", + "MemUsage": "11.22MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "495kB / 574kB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821882.998152, + "phase": "worker_short_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.84%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.10%", + "MemUsage": "67.09MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "327MB / 485MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.79%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.52%", + "MemUsage": "74.34MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "311MB / 476MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.67%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.65%", + "MemUsage": "14.92MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "26.6MB / 61.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.45%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.42%", + "MemUsage": "93.22MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "258MB / 93.4MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.18%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.34%", + "MemUsage": "13.7MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "2.46MB / 2.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821911.283901, + "phase": "db_short_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.27%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.07%", + "MemUsage": "66.9MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "328MB / 487MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.50%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.45%", + "MemUsage": "73.97MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "313MB / 479MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.18%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.68%", + "MemUsage": "14.95MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "26.7MB / 62.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.92%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.80%", + "MemUsage": "94.21MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "258MB / 93.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.12%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.47%", + "MemUsage": "15.09MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "4.17MB / 4.93MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821916.397165, + "phase": "db_short_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.02%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.13%", + "MemUsage": "67.25MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "329MB / 488MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.84%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.35%", + "MemUsage": "73.48MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "313MB / 479MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.76%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.69%", + "MemUsage": "14.96MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "26.8MB / 62.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.45%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.05%", + "MemUsage": "94.85MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "259MB / 93.8MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.47%", + "MemUsage": "15.09MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "4.42MB / 5.23MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794821948.858173, + "phase": "db_short_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.87%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.32%", + "MemUsage": "68.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "331MB / 491MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.87%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.55%", + "MemUsage": "74.52MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "316MB / 483MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.94%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.71%", + "MemUsage": "14.99MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "26.9MB / 62.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.41%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.92%", + "MemUsage": "97.09MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "260MB / 94.2MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.09%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.64%", + "MemUsage": "16.81MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "6.49MB / 7.55MB", + "PIDs": "10" + } + ] + }, + { + "at": 1794821976.812719, + "phase": "worker_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.66%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.36%", + "MemUsage": "68.38MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "333MB / 494MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.09%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.70%", + "MemUsage": "75.28MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "318MB / 486MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.87%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.61%", + "MemUsage": "14.86MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "27.1MB / 62.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.48%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.50%", + "MemUsage": "96.01MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "261MB / 94.5MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794822009.286432, + "phase": "worker_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.73%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.38%", + "MemUsage": "68.5MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "334MB / 497MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.15%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.64%", + "MemUsage": "74.95MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "320MB / 489MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.97%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.62%", + "MemUsage": "14.88MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "27.3MB / 63.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.50%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.82%", + "MemUsage": "96.82MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "262MB / 94.8MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794822041.765972, + "phase": "worker_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.74%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.40%", + "MemUsage": "68.59MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "337MB / 500MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.94%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.69%", + "MemUsage": "75.21MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "322MB / 493MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.88%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.64%", + "MemUsage": "14.89MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "27.5MB / 63.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.41%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.07%", + "MemUsage": "94.91MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "263MB / 95.2MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794822074.012128, + "phase": "worker_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.41%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.48%", + "MemUsage": "68.99MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "339MB / 504MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.75%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.66%", + "MemUsage": "75.08MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "324MB / 496MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.79%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.63%", + "MemUsage": "14.89MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "27.7MB / 64.1MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.47%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.51%", + "MemUsage": "93.47MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "264MB / 95.5MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794822106.4757571, + "phase": "worker_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.45%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.40%", + "MemUsage": "68.62MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "341MB / 507MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.81%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.38%", + "MemUsage": "73.63MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "326MB / 499MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.90%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.66%", + "MemUsage": "14.92MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "27.9MB / 64.5MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.40%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.24%", + "MemUsage": "92.79MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "265MB / 95.9MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794822138.986065, + "phase": "worker_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.16%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.44%", + "MemUsage": "68.81MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "343MB / 511MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.66%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.66%", + "MemUsage": "75.07MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "328MB / 502MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.86%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.66%", + "MemUsage": "14.92MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "28.1MB / 64.9MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.44%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.49%", + "MemUsage": "95.98MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "266MB / 96.2MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794822171.507664, + "phase": "worker_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.94%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.40%", + "MemUsage": "68.62MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "346MB / 514MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.35%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.39%", + "MemUsage": "73.68MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "330MB / 505MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.84%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.66%", + "MemUsage": "14.92MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "28.2MB / 65.3MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.45%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.04%", + "MemUsage": "94.81MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "267MB / 96.6MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794822204.027456, + "phase": "worker_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.60%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.33%", + "MemUsage": "68.27MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "348MB / 518MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.75%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.37%", + "MemUsage": "73.56MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "332MB / 509MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.68%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.87%", + "MemUsage": "15.2MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "28.4MB / 65.7MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.32%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.09%", + "MemUsage": "94.95MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "268MB / 96.9MB", + "PIDs": "20" + } + ] + }, + { + "at": 1794822217.053268, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.02%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.38%", + "MemUsage": "68.53MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "348MB / 519MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.11%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.67%", + "MemUsage": "75.11MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "333MB / 510MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.98%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.67%", + "MemUsage": "14.93MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "28.5MB / 65.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.46%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.39%", + "MemUsage": "95.72MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "268MB / 97.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "99.79%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.79%", + "MemUsage": "18.36MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "8.79MB / 9.9MB", + "PIDs": "11" + } + ] + }, + { + "at": 1794822249.52965, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.73%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.43%", + "MemUsage": "68.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "350MB / 522MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.63%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.79%", + "MemUsage": "75.72MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "335MB / 514MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.65%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.67%", + "MemUsage": "14.93MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "28.7MB / 66.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.15%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.39%", + "MemUsage": "98.27MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "270MB / 97.6MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.42%", + "MemUsage": "14.56MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "16.6MB / 19.2MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822282.057112, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.03%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.47%", + "MemUsage": "68.94MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "353MB / 525MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.09%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.35%", + "MemUsage": "73.47MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "337MB / 517MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.91%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.67%", + "MemUsage": "14.93MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "28.9MB / 66.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.00%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.22%", + "MemUsage": "97.85MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "271MB / 98MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.30%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.43%", + "MemUsage": "14.64MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "18.7MB / 21.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822314.045132, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.85%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.19%", + "MemUsage": "67.53MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "355MB / 529MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.75%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.70%", + "MemUsage": "75.25MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "339MB / 520MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.01%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.87%", + "MemUsage": "15.2MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "29MB / 67MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.48%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.60%", + "MemUsage": "93.7MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "272MB / 98.3MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.04%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.44%", + "MemUsage": "14.7MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "20.6MB / 24MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822346.500994, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.62%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.27%", + "MemUsage": "67.93MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "357MB / 532MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.17%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.70%", + "MemUsage": "75.27MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "341MB / 523MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.58%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.63%", + "MemUsage": "14.89MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "29.2MB / 67.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.34%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.59%", + "MemUsage": "96.22MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "273MB / 98.7MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.39%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.45%", + "MemUsage": "14.86MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "22.6MB / 26.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822378.980829, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.82%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.44%", + "MemUsage": "68.81MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "360MB / 536MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.54%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.73%", + "MemUsage": "75.41MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "343MB / 526MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.65%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.63%", + "MemUsage": "14.89MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "29.4MB / 67.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.35%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.85%", + "MemUsage": "96.91MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "274MB / 99.1MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.55%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.44%", + "MemUsage": "14.79MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "24.5MB / 28.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822411.46467, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.25%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.21%", + "MemUsage": "67.66MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "362MB / 539MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.86%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.66%", + "MemUsage": "75.07MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "345MB / 529MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.93%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "12.05%", + "MemUsage": "15.43MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "29.5MB / 68.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.48%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.32%", + "MemUsage": "95.55MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "275MB / 99.5MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.55%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.45%", + "MemUsage": "14.81MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "26.5MB / 31MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822443.055858, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.79%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.47%", + "MemUsage": "68.99MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "364MB / 542MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.85%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.75%", + "MemUsage": "75.5MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "347MB / 533MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.79%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.89%", + "MemUsage": "15.22MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "29.7MB / 68.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.38%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.33%", + "MemUsage": "95.55MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "276MB / 99.8MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.19%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.45%", + "MemUsage": "14.86MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "28.5MB / 33.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822475.494529, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.84%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.48%", + "MemUsage": "69.02MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "366MB / 546MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.15%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.70%", + "MemUsage": "75.26MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "349MB / 536MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.83%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.73%", + "MemUsage": "15.02MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "29.9MB / 69MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.61%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.34%", + "MemUsage": "95.6MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "278MB / 100MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.48%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.46%", + "MemUsage": "14.93MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "30.4MB / 35.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822507.99947, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.02%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.46%", + "MemUsage": "68.9MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "368MB / 549MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.14%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.51%", + "MemUsage": "74.31MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "352MB / 540MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.03%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.73%", + "MemUsage": "15.02MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "30MB / 69.4MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.81%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.55%", + "MemUsage": "96.13MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "279MB / 101MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.13%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.46%", + "MemUsage": "14.93MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "32.4MB / 38MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822540.486738, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.47%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.21%", + "MemUsage": "67.62MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "370MB / 553MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.58%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.71%", + "MemUsage": "75.31MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "353MB / 543MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.69%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.85%", + "MemUsage": "15.17MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "30.2MB / 69.8MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "1.50%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.55%", + "MemUsage": "96.13MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "280MB / 101MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.02%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.46%", + "MemUsage": "14.97MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "34.4MB / 40.4MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822572.056618, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.54%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.23%", + "MemUsage": "67.75MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "373MB / 557MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.98%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.71%", + "MemUsage": "75.33MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "355MB / 546MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.83%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.77%", + "MemUsage": "15.07MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "30.4MB / 70.2MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.70%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.79%", + "MemUsage": "96.75MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "281MB / 101MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.54%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.47%", + "MemUsage": "15.05MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "36.4MB / 42.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822604.088149, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.73%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.47%", + "MemUsage": "68.95MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "375MB / 560MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.16%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.72%", + "MemUsage": "75.35MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "357MB / 549MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.74%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.74%", + "MemUsage": "15.02MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "30.6MB / 70.6MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.39%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "39.25%", + "MemUsage": "100.5MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "282MB / 102MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.53%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.47%", + "MemUsage": "15.05MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "38.4MB / 45.1MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822636.5880089, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.78%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.54%", + "MemUsage": "69.34MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "377MB / 563MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.68%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.69%", + "MemUsage": "75.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "359MB / 552MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.82%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.73%", + "MemUsage": "15.02MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "30.7MB / 71MB", + "PIDs": "8" + }, + { + "BlockIO": "5.98MB / 0B", + "CPUPerc": "0.49%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.53%", + "MemUsage": "98.64MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "283MB / 102MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.41%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.47%", + "MemUsage": "15.09MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "40.3MB / 47.4MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822669.0715232, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.63%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.52%", + "MemUsage": "69.23MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "379MB / 566MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.39%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.75%", + "MemUsage": "75.5MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "361MB / 555MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.62%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.79%", + "MemUsage": "15.09MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "30.9MB / 71.4MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.47%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.13%", + "MemUsage": "97.62MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "284MB / 102MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.44%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.47%", + "MemUsage": "15.09MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "42.3MB / 49.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822701.620594, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.34%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.46%", + "MemUsage": "68.94MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "381MB / 570MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.73%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.70%", + "MemUsage": "75.26MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "363MB / 559MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.88%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.97%", + "MemUsage": "15.32MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "31.1MB / 71.7MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.45%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.59%", + "MemUsage": "98.78MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "285MB / 103MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.78%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.47%", + "MemUsage": "15.1MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "44.2MB / 52MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822733.080838, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.98%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.47%", + "MemUsage": "68.99MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "383MB / 573MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.93%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.72%", + "MemUsage": "75.37MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "365MB / 562MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.64%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "12.08%", + "MemUsage": "15.47MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "31.2MB / 72.1MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.74%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.55%", + "MemUsage": "96.13MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "286MB / 103MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.17%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.48%", + "MemUsage": "15.12MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "46.2MB / 54.4MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822765.575559, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.63%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.50%", + "MemUsage": "69.11MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "385MB / 576MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.61%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.50%", + "MemUsage": "74.25MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "368MB / 566MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.82%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.74%", + "MemUsage": "15.03MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "31.4MB / 72.5MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.29%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.70%", + "MemUsage": "93.95MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "288MB / 104MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.13%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.48%", + "MemUsage": "15.15MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "48.2MB / 56.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822798.03036, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.72%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.52%", + "MemUsage": "69.21MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "387MB / 579MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.90%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.74%", + "MemUsage": "75.45MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "370MB / 569MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.86%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.74%", + "MemUsage": "15.03MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "31.6MB / 72.9MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.41%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.08%", + "MemUsage": "92.36MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "289MB / 104MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.45%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.48%", + "MemUsage": "15.17MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "50.2MB / 59.2MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822829.571456, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.14%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.47%", + "MemUsage": "68.98MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "389MB / 582MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.84%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.75%", + "MemUsage": "75.51MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "372MB / 573MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.87%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.74%", + "MemUsage": "15.03MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "31.7MB / 73.3MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "2.03%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.28%", + "MemUsage": "98.01MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "290MB / 104MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.09%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.48%", + "MemUsage": "15.19MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "52.1MB / 61.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822862.1022642, + "phase": "worker_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.81%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.51%", + "MemUsage": "69.15MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "391MB / 585MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.65%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.74%", + "MemUsage": "75.48MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "375MB / 576MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.88%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.74%", + "MemUsage": "15.03MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "31.9MB / 73.7MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.47%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.71%", + "MemUsage": "99.11MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "291MB / 105MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.43%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.23MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "54.1MB / 63.8MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822877.228329, + "phase": "db_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.74%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.53%", + "MemUsage": "69.25MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "392MB / 587MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.23%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.53%", + "MemUsage": "74.38MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "376MB / 578MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.03%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.74%", + "MemUsage": "15.03MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "32MB / 73.9MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.77%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "39.23%", + "MemUsage": "100.4MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "291MB / 105MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.39%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.23MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "55MB / 64.8MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822909.716589, + "phase": "db_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.02%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.57%", + "MemUsage": "69.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "393MB / 589MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.68%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.79%", + "MemUsage": "75.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "377MB / 579MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.85%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.75%", + "MemUsage": "15.04MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "32.1MB / 74.3MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "1.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.76%", + "MemUsage": "99.22MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "293MB / 105MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.40%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.25MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "57MB / 67.1MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822942.125124, + "phase": "db_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.18%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.54%", + "MemUsage": "69.31MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "395MB / 591MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.65%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.51%", + "MemUsage": "74.31MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "378MB / 581MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.16%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.88%", + "MemUsage": "15.2MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "32.3MB / 74.7MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.59%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.06%", + "MemUsage": "97.45MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "294MB / 106MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.82%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.49%", + "MemUsage": "15.29MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "59MB / 69.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794822974.122357, + "phase": "db_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.95%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.60%", + "MemUsage": "69.63MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "396MB / 593MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.49%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.70%", + "MemUsage": "75.29MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "381MB / 584MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.29%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.90%", + "MemUsage": "15.23MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "32.5MB / 75.1MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.71%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.22%", + "MemUsage": "92.71MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "295MB / 106MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.39%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.50%", + "MemUsage": "15.36MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "61MB / 71.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823006.591758, + "phase": "db_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.81%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.29%", + "MemUsage": "68.06MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "398MB / 595MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.08%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.88%", + "MemUsage": "76.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "382MB / 585MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.81%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.80%", + "MemUsage": "15.11MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "32.6MB / 75.5MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.38%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.34%", + "MemUsage": "93.02MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "296MB / 107MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.98%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.50%", + "MemUsage": "15.38MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "63MB / 74.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823039.1302888, + "phase": "db_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.33%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.49%", + "MemUsage": "69.08MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "399MB / 596MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.27%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.69%", + "MemUsage": "75.19MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "384MB / 588MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.53%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.80%", + "MemUsage": "15.1MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "32.8MB / 75.9MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.41%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.67%", + "MemUsage": "93.89MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "298MB / 107MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.18%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.50%", + "MemUsage": "15.39MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "64.9MB / 76.6MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823071.638259, + "phase": "db_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.46%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.38%", + "MemUsage": "68.53MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "401MB / 598MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.12%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.68%", + "MemUsage": "75.16MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "385MB / 590MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.64%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.80%", + "MemUsage": "15.11MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "33MB / 76.3MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.48%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.22%", + "MemUsage": "95.27MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "299MB / 107MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.41MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "66.9MB / 78.9MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823103.804856, + "phase": "db_long_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.95%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.76%", + "MemUsage": "70.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "403MB / 600MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "3.45%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.03%", + "MemUsage": "76.96MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "387MB / 592MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.05%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.80%", + "MemUsage": "15.1MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "33.2MB / 76.7MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "2.80%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.43%", + "MemUsage": "95.82MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "300MB / 108MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.54%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.45MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "68.8MB / 81.2MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823117.4464839, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.77%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.61%", + "MemUsage": "69.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "403MB / 601MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.10%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.92%", + "MemUsage": "76.39MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "388MB / 593MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.22%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "12.08%", + "MemUsage": "15.47MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "33.2MB / 76.8MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.43%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.44%", + "MemUsage": "95.84MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "301MB / 108MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.18%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.45MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "69.7MB / 82.3MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823149.9834082, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.27%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.65%", + "MemUsage": "69.87MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "406MB / 605MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.66%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.88%", + "MemUsage": "76.21MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "390MB / 596MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.63%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.70%", + "MemUsage": "14.98MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "33.4MB / 77.2MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.90%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "37.75%", + "MemUsage": "96.63MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "302MB / 108MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.19%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.48MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "71.8MB / 84.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823182.3198771, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.03%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.65%", + "MemUsage": "69.9MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "407MB / 608MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "3.62%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.77%", + "MemUsage": "75.61MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "392MB / 600MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.11%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.89%", + "MemUsage": "15.21MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "33.6MB / 77.6MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "1.27%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "38.15%", + "MemUsage": "97.67MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "303MB / 109MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "5.23%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.49MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "73.7MB / 87MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823214.352156, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.71%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.38%", + "MemUsage": "68.51MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "410MB / 611MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.89%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.05%", + "MemUsage": "77.07MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "394MB / 603MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.94%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.79%", + "MemUsage": "15.09MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "33.7MB / 78MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.78%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "31.43%", + "MemUsage": "80.46MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "304MB / 109MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.12%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.51MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "75.7MB / 89.4MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823247.024501, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.99%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.66%", + "MemUsage": "69.95MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "412MB / 615MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.39%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.98%", + "MemUsage": "76.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "396MB / 606MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.24%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.79%", + "MemUsage": "15.09MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "33.9MB / 78.4MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "1.93%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "31.60%", + "MemUsage": "80.89MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "305MB / 109MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.48%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.51%", + "MemUsage": "15.51MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "77.7MB / 91.7MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823280.058184, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.26%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.55%", + "MemUsage": "69.35MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "414MB / 619MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.76%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.96%", + "MemUsage": "76.61MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "398MB / 609MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.02%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.75%", + "MemUsage": "15.04MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "34.1MB / 78.8MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "2.24%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "31.60%", + "MemUsage": "80.89MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "306MB / 110MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.77%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.52MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "79.6MB / 94MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823312.168788, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.09%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.65%", + "MemUsage": "69.91MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "417MB / 622MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.82%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.95%", + "MemUsage": "76.55MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "400MB / 612MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.03%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.75%", + "MemUsage": "15.04MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "34.2MB / 79.2MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.92%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "31.60%", + "MemUsage": "80.89MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "307MB / 110MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.25%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.66MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "81.7MB / 96.5MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823344.273514, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.40%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.70%", + "MemUsage": "70.13MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "419MB / 626MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.39%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.03%", + "MemUsage": "76.95MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "402MB / 615MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.82%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.75%", + "MemUsage": "15.04MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "34.4MB / 79.6MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.67%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.09%", + "MemUsage": "82.15MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "308MB / 111MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.96%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.56MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "83.6MB / 98.8MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823376.909882, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.61%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.67%", + "MemUsage": "69.99MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "422MB / 629MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.93%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.97%", + "MemUsage": "76.65MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "403MB / 618MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.69%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.75%", + "MemUsage": "15.04MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "34.6MB / 79.9MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.62%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.96%", + "MemUsage": "84.38MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "310MB / 111MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.27%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.57MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "85.6MB / 101MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823409.4744241, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.00%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.68%", + "MemUsage": "70.06MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "424MB / 633MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.41%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.79%", + "MemUsage": "75.74MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "405MB / 621MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.79%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.75%", + "MemUsage": "15.04MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "34.7MB / 80.3MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "2.07%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "33.74%", + "MemUsage": "86.38MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "311MB / 111MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.28%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.59MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "87.6MB / 103MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823442.148005, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.75%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.67%", + "MemUsage": "69.99MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "426MB / 636MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.02%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.97%", + "MemUsage": "76.63MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "407MB / 624MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.92%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.75%", + "MemUsage": "15.04MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "34.9MB / 80.7MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.89%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.73%", + "MemUsage": "83.79MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "312MB / 112MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.62%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.52%", + "MemUsage": "15.61MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "89.6MB / 106MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823474.20546, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.24%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.68%", + "MemUsage": "70.02MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "428MB / 640MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "2.57%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.95%", + "MemUsage": "76.54MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "409MB / 628MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.79%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.75%", + "MemUsage": "15.04MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "35.1MB / 81.1MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.44%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.64%", + "MemUsage": "83.56MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "313MB / 112MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.98%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.63MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "91.6MB / 108MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823507.008907, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.18%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.69%", + "MemUsage": "70.08MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "430MB / 643MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.07%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.01%", + "MemUsage": "76.87MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "412MB / 631MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.79%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.90%", + "MemUsage": "15.24MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "35.2MB / 81.5MB", + "PIDs": "8" + }, + { + "BlockIO": "6.11MB / 0B", + "CPUPerc": "0.74%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "32.34%", + "MemUsage": "82.8MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "314MB / 112MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.10%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.64MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "93.6MB / 111MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823539.727211, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.84%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.73%", + "MemUsage": "70.29MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "432MB / 646MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.98%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.06%", + "MemUsage": "77.09MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "414MB / 635MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.83%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.87%", + "MemUsage": "15.19MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "35.4MB / 81.9MB", + "PIDs": "8" + }, + { + "BlockIO": "6.24MB / 0B", + "CPUPerc": "1.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.13%", + "MemUsage": "89.95MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "315MB / 113MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.63MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "95.5MB / 113MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823572.206418, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.75%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.68%", + "MemUsage": "70.06MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "433MB / 649MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.36%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.95%", + "MemUsage": "76.54MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "417MB / 639MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.75%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.72%", + "MemUsage": "15.01MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "35.6MB / 82.3MB", + "PIDs": "8" + }, + { + "BlockIO": "6.24MB / 0B", + "CPUPerc": "0.61%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.49%", + "MemUsage": "90.86MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "316MB / 113MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.60%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.66MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "97.6MB / 115MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823604.2859259, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.75%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.70%", + "MemUsage": "70.15MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "436MB / 653MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.26%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.97%", + "MemUsage": "76.66MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "418MB / 642MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.71%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.73%", + "MemUsage": "15.01MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "35.7MB / 82.7MB", + "PIDs": "8" + }, + { + "BlockIO": "6.24MB / 0B", + "CPUPerc": "1.24%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.54%", + "MemUsage": "90.98MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "317MB / 114MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.31%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.79MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "99.6MB / 118MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823636.856317, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.95%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.69%", + "MemUsage": "70.11MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "438MB / 656MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.40%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.95%", + "MemUsage": "76.54MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "421MB / 645MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.63%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.93%", + "MemUsage": "15.27MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "35.9MB / 83.1MB", + "PIDs": "8" + }, + { + "BlockIO": "6.24MB / 0B", + "CPUPerc": "0.77%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.18%", + "MemUsage": "92.61MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "319MB / 114MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.81%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.85%", + "MemUsage": "18.96MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "102MB / 120MB", + "PIDs": "10" + } + ] + }, + { + "at": 1794823669.4016461, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.64%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.41%", + "MemUsage": "68.65MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "440MB / 659MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.46%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.74%", + "MemUsage": "75.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "423MB / 649MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.56%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.86%", + "MemUsage": "15.18MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "36.1MB / 83.5MB", + "PIDs": "8" + }, + { + "BlockIO": "6.24MB / 0B", + "CPUPerc": "0.70%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.11%", + "MemUsage": "92.44MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "320MB / 114MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.06%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.68MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "103MB / 122MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823702.0271401, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.28%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.46%", + "MemUsage": "68.9MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "442MB / 662MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.55%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.96%", + "MemUsage": "76.57MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "425MB / 652MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.85%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.71%", + "MemUsage": "14.99MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "36.2MB / 83.9MB", + "PIDs": "8" + }, + { + "BlockIO": "6.24MB / 0B", + "CPUPerc": "0.64%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.62%", + "MemUsage": "91.2MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "321MB / 115MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.69MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "105MB / 125MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823734.2180429, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.94%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.49%", + "MemUsage": "69.07MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "444MB / 665MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.60%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.98%", + "MemUsage": "76.68MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "427MB / 656MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.74%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.71%", + "MemUsage": "14.99MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "36.4MB / 84.3MB", + "PIDs": "8" + }, + { + "BlockIO": "6.24MB / 0B", + "CPUPerc": "0.68%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.99%", + "MemUsage": "89.57MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "322MB / 115MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.27%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.53%", + "MemUsage": "15.7MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "108MB / 127MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823766.846899, + "phase": "db_recovery_3", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.70%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.74%", + "MemUsage": "70.37MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "446MB / 669MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.17%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.02%", + "MemUsage": "76.9MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "430MB / 659MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.73%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.80%", + "MemUsage": "15.1MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "36.6MB / 84.7MB", + "PIDs": "8" + }, + { + "BlockIO": "6.24MB / 0B", + "CPUPerc": "0.54%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.37%", + "MemUsage": "87.99MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "323MB / 115MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.75%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.72MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "109MB / 129MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823777.973217, + "phase": "tempo_unavailable", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.66%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.70%", + "MemUsage": "70.12MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "447MB / 670MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.27%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.00%", + "MemUsage": "76.8MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "430MB / 660MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.74%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.90%", + "MemUsage": "15.24MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "36.6MB / 84.8MB", + "PIDs": "8" + }, + { + "BlockIO": "6.24MB / 0B", + "CPUPerc": "0.50%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.37%", + "MemUsage": "87.99MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "323MB / 116MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.22%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.73MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "110MB / 130MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823810.56732, + "phase": "tempo_unavailable", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.80%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.45%", + "MemUsage": "68.88MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "449MB / 673MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.97%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.01%", + "MemUsage": "76.88MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "432MB / 663MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.95%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.71%", + "MemUsage": "14.99MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "36.8MB / 85.2MB", + "PIDs": "8" + }, + { + "BlockIO": "6.37MB / 0B", + "CPUPerc": "0.61%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.47%", + "MemUsage": "88.24MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "325MB / 116MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.15%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.73MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "112MB / 133MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823842.207754, + "phase": "tempo_unavailable", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.40%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.71%", + "MemUsage": "70.22MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "451MB / 676MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.83%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.98%", + "MemUsage": "76.7MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "434MB / 667MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.76%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.71%", + "MemUsage": "14.99MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "37MB / 85.6MB", + "PIDs": "8" + }, + { + "BlockIO": "6.37MB / 0B", + "CPUPerc": "0.52%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.91%", + "MemUsage": "89.37MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "326MB / 116MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.24%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.75MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "114MB / 135MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823874.31151, + "phase": "tempo_unavailable", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.42%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.72%", + "MemUsage": "70.25MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "453MB / 680MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.38%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.94%", + "MemUsage": "76.48MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "437MB / 670MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.09%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.80%", + "MemUsage": "15.1MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "37.1MB / 86MB", + "PIDs": "8" + }, + { + "BlockIO": "6.37MB / 0B", + "CPUPerc": "1.24%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.60%", + "MemUsage": "91.12MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "327MB / 116MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.32%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.77MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "116MB / 137MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823907.22252, + "phase": "tempo_unavailable", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.03%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.76%", + "MemUsage": "70.46MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "455MB / 683MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.55%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.04%", + "MemUsage": "77.02MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "439MB / 674MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.62%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.80%", + "MemUsage": "15.1MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "37.3MB / 86.4MB", + "PIDs": "8" + }, + { + "BlockIO": "6.37MB / 0B", + "CPUPerc": "0.42%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.61%", + "MemUsage": "91.16MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "328MB / 116MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.09%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.77MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "118MB / 140MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823939.8670669, + "phase": "tempo_unavailable", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.25%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.73%", + "MemUsage": "70.31MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "457MB / 686MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.72%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.93%", + "MemUsage": "76.45MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "441MB / 677MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.76%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.76%", + "MemUsage": "15.05MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "37.5MB / 86.8MB", + "PIDs": "8" + }, + { + "BlockIO": "6.37MB / 0B", + "CPUPerc": "2.94%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.43%", + "MemUsage": "90.7MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "329MB / 117MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.54%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.78MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "120MB / 142MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794823972.30054, + "phase": "tempo_unavailable", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.79%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.73%", + "MemUsage": "70.29MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "460MB / 690MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.52%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.02%", + "MemUsage": "76.91MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "443MB / 680MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.81%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.77%", + "MemUsage": "15.07MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "37.6MB / 87.2MB", + "PIDs": "8" + }, + { + "BlockIO": "6.37MB / 0B", + "CPUPerc": "0.60%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.76%", + "MemUsage": "88.98MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "330MB / 117MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.45%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.79MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "122MB / 144MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794824004.2541828, + "phase": "tempo_unavailable", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.74%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.63%", + "MemUsage": "69.77MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "462MB / 694MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.85%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.02%", + "MemUsage": "76.92MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "444MB / 683MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.09%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.77%", + "MemUsage": "15.06MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "37.8MB / 87.6MB", + "PIDs": "8" + }, + { + "BlockIO": "6.37MB / 0B", + "CPUPerc": "1.52%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "34.85%", + "MemUsage": "89.22MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "331MB / 117MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.42%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.8MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "124MB / 147MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794824036.9491751, + "phase": "tempo_unavailable", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.86%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.58%", + "MemUsage": "69.51MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "465MB / 697MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.71%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.03%", + "MemUsage": "76.96MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "446MB / 686MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.62%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.69%", + "MemUsage": "14.96MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "38MB / 88MB", + "PIDs": "8" + }, + { + "BlockIO": "6.47MB / 0B", + "CPUPerc": "0.61%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.29%", + "MemUsage": "90.33MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "332MB / 117MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.05%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.81MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "126MB / 149MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794824069.663313, + "phase": "tempo_unavailable", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.69%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.72%", + "MemUsage": "70.23MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "467MB / 700MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.22%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "14.99%", + "MemUsage": "76.75MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "448MB / 689MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.94%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.69%", + "MemUsage": "14.96MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "38.2MB / 88.4MB", + "PIDs": "8" + }, + { + "BlockIO": "6.47MB / 0B", + "CPUPerc": "3.51%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.45%", + "MemUsage": "93.32MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "333MB / 117MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.14%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.81MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "128MB / 151MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794824078.342614, + "phase": "tempo_recovery", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.82%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.76%", + "MemUsage": "70.43MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "467MB / 702MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "1.70%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.08%", + "MemUsage": "77.2MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "449MB / 690MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.24%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.69%", + "MemUsage": "14.96MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "38.2MB / 88.5MB", + "PIDs": "8" + }, + { + "BlockIO": "6.47MB / 0B", + "CPUPerc": "0.57%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.66%", + "MemUsage": "93.84MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "334MB / 118MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.83%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.54%", + "MemUsage": "15.82MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "129MB / 152MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794824110.984055, + "phase": "tempo_recovery", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.72%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.77%", + "MemUsage": "70.48MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "469MB / 705MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.77%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.00%", + "MemUsage": "76.82MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "451MB / 694MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.63%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.69%", + "MemUsage": "14.96MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "38.4MB / 88.9MB", + "PIDs": "8" + }, + { + "BlockIO": "6.47MB / 0B", + "CPUPerc": "0.40%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "35.60%", + "MemUsage": "91.15MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "335MB / 118MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.29%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.56%", + "MemUsage": "15.94MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "131MB / 154MB", + "PIDs": "9" + } + ] + }, + { + "at": 1794824138.494708, + "phase": "final_drain", + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "8219f2431973bb3309aa7fef5aa8717c545732c2891515ba01ce5b56b52a2d32", + "ID": "8219f2431973", + "MemPerc": "13.74%", + "MemUsage": "70.35MiB / 512MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "471MB / 707MB", + "PIDs": "9" + }, + { + "BlockIO": "463kB / 0B", + "CPUPerc": "0.15%", + "Container": "a5026af5cc2c5e2b4786ca56e8715c5863b84f12c74cca7b65af40ca747ee58f", + "ID": "a5026af5cc2c", + "MemPerc": "15.09%", + "MemUsage": "77.29MiB / 512MiB", + "Name": "mtusi2026lab6-api-2", + "NetIO": "453MB / 697MB", + "PIDs": "9" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.90%", + "Container": "09c8099ad53db6a3c7a3db3e2aaf8c862f28f0c7153811ff76cbbfd1eaa45eb4", + "ID": "09c8099ad53d", + "MemPerc": "11.89%", + "MemUsage": "15.22MiB / 128MiB", + "Name": "mtusi2026lab6-dispatcher-1", + "NetIO": "38.5MB / 89.2MB", + "PIDs": "8" + }, + { + "BlockIO": "6.47MB / 0B", + "CPUPerc": "0.31%", + "Container": "43b7b11d0b3b0c5f14fa498f7a98343690befffb7a7a63eb041f87bfd680c4d6", + "ID": "43b7b11d0b3b", + "MemPerc": "36.04%", + "MemUsage": "92.26MiB / 256MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "336MB / 118MB", + "PIDs": "20" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.08%", + "Container": "554a3f3722680e813be5531d9b12a506842c6fb710ed513083a6d8024069e3f2", + "ID": "554a3f372268", + "MemPerc": "1.55%", + "MemUsage": "15.84MiB / 1GiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "132MB / 156MB", + "PIDs": "9" + } + ] + } + ], + "phases": [ + { + "name": "baseline", + "start": 1794817079.719438, + "end": 1794817979.729019, + "before": { + "requests": 0, + "success": 0, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 0, + "latency_success_ms": { + "0.5": null, + "0.95": null, + "0.99": null + }, + "statuses": {} + }, + "after": { + "requests": 9451, + "success": 9451, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 450, + "latency_success_ms": { + "0.5": 8.373958058655262, + "0.95": 16.348457895219326, + "0.99": 25.746542029082775 + }, + "statuses": { + "200": 8100, + "201": 901, + "202": 450 + } + }, + "successful_rps": 10.50100502932713, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.373958058655262, + "0.95": 16.348457895219326, + "0.99": 25.746542029082775 + }, + "statuses": { + "200": 8100, + "201": 901, + "202": 450 + } + }, + { + "name": "worker_short_1", + "start": 1794817980.084564, + "end": 1794817985.0889568, + "before": { + "requests": 9455, + "success": 9455, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 451, + "latency_success_ms": { + "0.5": 8.373958058655262, + "0.95": 16.35108282789588, + "0.99": 25.799499824643135 + }, + "statuses": { + "200": 8103, + "201": 901, + "202": 451 + } + }, + "after": { + "requests": 9507, + "success": 9507, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 453, + "latency_success_ms": { + "0.5": 8.370500057935715, + "0.95": 16.348457895219326, + "0.99": 25.746542029082775 + }, + "statuses": { + "200": 8148, + "201": 906, + "202": 453 + } + }, + "successful_rps": 10.3908971041871, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 7.937416899949312, + "0.95": 16.20679209008813, + "0.99": 17.71804178133607 + }, + "statuses": { + "200": 45, + "201": 5, + "202": 2 + } + }, + { + "name": "worker_short_recovery_1", + "start": 1794817985.306079, + "end": 1794818045.3097181, + "before": { + "requests": 9510, + "success": 9510, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 453, + "latency_success_ms": { + "0.5": 8.370375260710716, + "0.95": 16.348457895219326, + "0.99": 25.746542029082775 + }, + "statuses": { + "200": 8151, + "201": 906, + "202": 453 + } + }, + "after": { + "requests": 10139, + "success": 10139, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 483, + "latency_success_ms": { + "0.5": 8.377707563340664, + "0.95": 16.47312520071864, + "0.99": 25.799499824643135 + }, + "statuses": { + "200": 8690, + "201": 966, + "202": 483 + } + }, + "successful_rps": 10.499373687911817, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.417167235165834, + "0.95": 17.10041705518961, + "0.99": 26.179499924182892 + }, + "statuses": { + "200": 540, + "201": 60, + "202": 30 + } + }, + { + "name": "db_short_1", + "start": 1794818045.434122, + "end": 1794818050.440675, + "before": { + "requests": 10141, + "success": 10141, + "errors": 0, + "generator_dropped": 0, + "uploads_accepted": 483, + "latency_success_ms": { + "0.5": 8.377707563340664, + "0.95": 16.47312520071864, + "0.99": 25.799499824643135 + }, + "statuses": { + "200": 8692, + "201": 966, + "202": 483 + } + }, + "after": { + "requests": 10174, + "success": 10144, + "errors": 30, + "generator_dropped": 0, + "uploads_accepted": 486, + "latency_success_ms": { + "0.5": 8.378332946449518, + "0.95": 16.507582738995552, + "0.99": 25.88783297687769 + }, + "statuses": { + "200": 8692, + "201": 966, + "202": 486, + "503": 30 + } + }, + "successful_rps": 0.5992161038489773, + "error_ratio": 0.9090909090909091, + "latency_success_ms": { + "0.5": 23.7681670114398, + "0.95": 27.764666825532913, + "0.99": 27.764666825532913 + }, + "statuses": { + "202": 3, + "503": 30 + } + }, + { + "name": "db_short_recovery_1", + "start": 1794818050.568737, + "end": 1794818110.5733361, + "before": { + "requests": 10195, + "success": 10165, + "errors": 30, + "generator_dropped": 0, + "uploads_accepted": 486, + "latency_success_ms": { + "0.5": 8.389250375330448, + "0.95": 16.710457857698202, + "0.99": 27.066375128924847 + }, + "statuses": { + "200": 8711, + "201": 968, + "202": 486, + "503": 30 + } + }, + "after": { + "requests": 10825, + "success": 10795, + "errors": 30, + "generator_dropped": 0, + "uploads_accepted": 516, + "latency_success_ms": { + "0.5": 8.428083267062902, + "0.95": 16.712582670152187, + "0.99": 27.063041925430298 + }, + "statuses": { + "200": 9251, + "201": 1028, + "202": 516, + "503": 30 + } + }, + "successful_rps": 10.499202103454953, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 9.02662519365549, + "0.95": 16.82770811021328, + "0.99": 26.06612490490079 + }, + "statuses": { + "200": 540, + "201": 60, + "202": 30 + } + }, + { + "name": "worker_long_1", + "start": 1794818110.87646, + "end": 1794818350.879679, + "before": { + "requests": 10828, + "success": 10798, + "errors": 30, + "generator_dropped": 0, + "uploads_accepted": 516, + "latency_success_ms": { + "0.5": 8.425625041127205, + "0.95": 16.712582670152187, + "0.99": 27.063041925430298 + }, + "statuses": { + "200": 9253, + "201": 1029, + "202": 516, + "503": 30 + } + }, + "after": { + "requests": 13348, + "success": 13318, + "errors": 30, + "generator_dropped": 0, + "uploads_accepted": 636, + "latency_success_ms": { + "0.5": 8.502291981130838, + "0.95": 16.82770811021328, + "0.99": 27.163791935890913 + }, + "statuses": { + "200": 11413, + "201": 1269, + "202": 636, + "503": 30 + } + }, + "successful_rps": 10.499865558974752, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.786541875451803, + "0.95": 17.55095785483718, + "0.99": 28.451249934732914 + }, + "statuses": { + "200": 2160, + "201": 240, + "202": 120 + } + }, + { + "name": "worker_recovery_1", + "start": 1794818351.155176, + "end": 1794819011.157911, + "before": { + "requests": 13351, + "success": 13321, + "errors": 30, + "generator_dropped": 0, + "uploads_accepted": 636, + "latency_success_ms": { + "0.5": 8.502583019435406, + "0.95": 16.826041974127293, + "0.99": 27.163791935890913 + }, + "statuses": { + "200": 11416, + "201": 1269, + "202": 636, + "503": 30 + } + }, + "after": { + "requests": 20281, + "success": 20251, + "errors": 30, + "generator_dropped": 0, + "uploads_accepted": 966, + "latency_success_ms": { + "0.5": 8.388000074774027, + "0.95": 16.76866691559553, + "0.99": 26.405250187963247 + }, + "statuses": { + "200": 17356, + "201": 1929, + "202": 966, + "503": 30 + } + }, + "successful_rps": 10.49996001789839, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.192374836653471, + "0.95": 16.595167107880116, + "0.99": 25.254250038415194 + }, + "statuses": { + "200": 5940, + "201": 660, + "202": 330 + } + }, + { + "name": "db_long_1", + "start": 1794819011.315764, + "end": 1794819251.318689, + "before": { + "requests": 20282, + "success": 20252, + "errors": 30, + "generator_dropped": 0, + "uploads_accepted": 966, + "latency_success_ms": { + "0.5": 8.388000074774027, + "0.95": 16.76866691559553, + "0.99": 26.405250187963247 + }, + "statuses": { + "200": 17357, + "201": 1929, + "202": 966, + "503": 30 + } + }, + "after": { + "requests": 22782, + "success": 20372, + "errors": 2410, + "generator_dropped": 0, + "uploads_accepted": 1086, + "latency_success_ms": { + "0.5": 8.428208064287901, + "0.95": 17.40583311766386, + "0.99": 28.451249934732914 + }, + "statuses": { + "200": 17357, + "201": 1929, + "202": 1086, + "503": 2410 + } + }, + "successful_rps": 0.49999409771972575, + "error_ratio": 0.952, + "latency_success_ms": { + "0.5": 27.061082888394594, + "0.95": 41.25562496483326, + "0.99": 48.944166861474514 + }, + "statuses": { + "202": 120, + "503": 2380 + } + }, + { + "name": "db_recovery_1", + "start": 1794819251.457268, + "end": 1794819911.460705, + "before": { + "requests": 22804, + "success": 20393, + "errors": 2411, + "generator_dropped": 0, + "uploads_accepted": 1086, + "latency_success_ms": { + "0.5": 8.435375057160854, + "0.95": 17.494166735559702, + "0.99": 29.01849988847971 + }, + "statuses": { + "200": 17376, + "201": 1931, + "202": 1086, + "503": 2411 + } + }, + "after": { + "requests": 29734, + "success": 27323, + "errors": 2411, + "generator_dropped": 0, + "uploads_accepted": 1416, + "latency_success_ms": { + "0.5": 8.369749877601862, + "0.95": 17.159542068839073, + "0.99": 27.726917061954737 + }, + "statuses": { + "200": 23316, + "201": 2591, + "202": 1416, + "503": 2411 + } + }, + "successful_rps": 10.499951904690572, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.165542036294937, + "0.95": 16.40924997627735, + "0.99": 24.354166816920042 + }, + "statuses": { + "200": 5940, + "201": 660, + "202": 330 + } + }, + { + "name": "worker_short_2", + "start": 1794819911.837276, + "end": 1794819916.8478432, + "before": { + "requests": 29739, + "success": 27328, + "errors": 2411, + "generator_dropped": 0, + "uploads_accepted": 1417, + "latency_success_ms": { + "0.5": 8.369707968086004, + "0.95": 17.159542068839073, + "0.99": 27.726917061954737 + }, + "statuses": { + "200": 23319, + "201": 2592, + "202": 1417, + "503": 2411 + } + }, + "after": { + "requests": 29791, + "success": 27380, + "errors": 2411, + "generator_dropped": 0, + "uploads_accepted": 1419, + "latency_success_ms": { + "0.5": 8.367832750082016, + "0.95": 17.159542068839073, + "0.99": 27.726917061954737 + }, + "statuses": { + "200": 23364, + "201": 2597, + "202": 1419, + "503": 2411 + } + }, + "successful_rps": 10.37807944264453, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 6.861874833703041, + "0.95": 17.471542116254568, + "0.99": 27.467207983136177 + }, + "statuses": { + "200": 45, + "201": 5, + "202": 2 + } + }, + { + "name": "worker_short_recovery_2", + "start": 1794819917.066106, + "end": 1794819977.075011, + "before": { + "requests": 29793, + "success": 27382, + "errors": 2411, + "generator_dropped": 0, + "uploads_accepted": 1419, + "latency_success_ms": { + "0.5": 8.367832750082016, + "0.95": 17.159542068839073, + "0.99": 27.726917061954737 + }, + "statuses": { + "200": 23366, + "201": 2597, + "202": 1419, + "503": 2411 + } + }, + "after": { + "requests": 30423, + "success": 28012, + "errors": 2411, + "generator_dropped": 0, + "uploads_accepted": 1449, + "latency_success_ms": { + "0.5": 8.365374989807606, + "0.95": 17.118082847446203, + "0.99": 27.59991679340601 + }, + "statuses": { + "200": 23906, + "201": 2657, + "202": 1449, + "503": 2411 + } + }, + "successful_rps": 10.498446956541944, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.114666678011417, + "0.95": 16.183000057935715, + "0.99": 24.04812490567565 + }, + "statuses": { + "200": 540, + "201": 60, + "202": 30 + } + }, + { + "name": "db_short_2", + "start": 1794819977.215701, + "end": 1794819982.219071, + "before": { + "requests": 30424, + "success": 28013, + "errors": 2411, + "generator_dropped": 0, + "uploads_accepted": 1449, + "latency_success_ms": { + "0.5": 8.365374989807606, + "0.95": 17.118082847446203, + "0.99": 27.59991679340601 + }, + "statuses": { + "200": 23907, + "201": 2657, + "202": 1449, + "503": 2411 + } + }, + "after": { + "requests": 30457, + "success": 28016, + "errors": 2441, + "generator_dropped": 0, + "uploads_accepted": 1452, + "latency_success_ms": { + "0.5": 8.36570793762803, + "0.95": 17.132124863564968, + "0.99": 27.726917061954737 + }, + "statuses": { + "200": 23907, + "201": 2657, + "202": 1452, + "503": 2441 + } + }, + "successful_rps": 0.5995964668020037, + "error_ratio": 0.9090909090909091, + "latency_success_ms": { + "0.5": 31.136707868427038, + "0.95": 34.025250002741814, + "0.99": 34.025250002741814 + }, + "statuses": { + "202": 3, + "503": 30 + } + }, + { + "name": "db_short_recovery_2", + "start": 1794819982.3592389, + "end": 1794820042.3702369, + "before": { + "requests": 30479, + "success": 28037, + "errors": 2442, + "generator_dropped": 0, + "uploads_accepted": 1452, + "latency_success_ms": { + "0.5": 8.369749877601862, + "0.95": 17.239124979823828, + "0.99": 28.457041829824448 + }, + "statuses": { + "200": 23926, + "201": 2659, + "202": 1452, + "503": 2442 + } + }, + "after": { + "requests": 31109, + "success": 28667, + "errors": 2442, + "generator_dropped": 0, + "uploads_accepted": 1482, + "latency_success_ms": { + "0.5": 8.363292086869478, + "0.95": 17.236500047147274, + "0.99": 28.451249934732914 + }, + "statuses": { + "200": 24466, + "201": 2719, + "202": 1482, + "503": 2442 + } + }, + "successful_rps": 10.49808212389292, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 7.915124762803316, + "0.95": 17.11429189890623, + "0.99": 27.86441706120968 + }, + "statuses": { + "200": 540, + "201": 60, + "202": 30 + } + }, + { + "name": "worker_long_2", + "start": 1794820042.693995, + "end": 1794820282.7011049, + "before": { + "requests": 31112, + "success": 28670, + "errors": 2442, + "generator_dropped": 0, + "uploads_accepted": 1482, + "latency_success_ms": { + "0.5": 8.363292086869478, + "0.95": 17.236500047147274, + "0.99": 28.451249934732914 + }, + "statuses": { + "200": 24469, + "201": 2719, + "202": 1482, + "503": 2442 + } + }, + "after": { + "requests": 33632, + "success": 31190, + "errors": 2442, + "generator_dropped": 0, + "uploads_accepted": 1602, + "latency_success_ms": { + "0.5": 8.346541784703732, + "0.95": 17.150333151221275, + "0.99": 28.16483285278082 + }, + "statuses": { + "200": 26629, + "201": 2959, + "202": 1602, + "503": 2442 + } + }, + "successful_rps": 10.499693843708734, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.250332903116941, + "0.95": 16.793041955679655, + "0.99": 24.823333136737347 + }, + "statuses": { + "200": 2160, + "201": 240, + "202": 120 + } + }, + { + "name": "worker_recovery_2", + "start": 1794820282.981466, + "end": 1794820942.98451, + "before": { + "requests": 33635, + "success": 31193, + "errors": 2442, + "generator_dropped": 0, + "uploads_accepted": 1602, + "latency_success_ms": { + "0.5": 8.346582762897015, + "0.95": 17.159542068839073, + "0.99": 28.16483285278082 + }, + "statuses": { + "200": 26631, + "201": 2960, + "202": 1602, + "503": 2442 + } + }, + "after": { + "requests": 40565, + "success": 38123, + "errors": 2442, + "generator_dropped": 0, + "uploads_accepted": 1932, + "latency_success_ms": { + "0.5": 8.33133328706026, + "0.95": 17.094874754548073, + "0.99": 27.571292128413916 + }, + "statuses": { + "200": 32571, + "201": 3620, + "202": 1932, + "503": 2442 + } + }, + "successful_rps": 10.499955663543126, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.26845783740282, + "0.95": 16.80166693404317, + "0.99": 26.104208081960678 + }, + "statuses": { + "200": 5940, + "201": 660, + "202": 330 + } + }, + { + "name": "db_long_2", + "start": 1794820943.135081, + "end": 1794821183.136804, + "before": { + "requests": 40566, + "success": 38124, + "errors": 2442, + "generator_dropped": 0, + "uploads_accepted": 1932, + "latency_success_ms": { + "0.5": 8.33129184320569, + "0.95": 17.094874754548073, + "0.99": 27.571292128413916 + }, + "statuses": { + "200": 32572, + "201": 3620, + "202": 1932, + "503": 2442 + } + }, + "after": { + "requests": 43066, + "success": 38244, + "errors": 4822, + "generator_dropped": 0, + "uploads_accepted": 2052, + "latency_success_ms": { + "0.5": 8.356708101928234, + "0.95": 17.484167125076056, + "0.99": 28.96799985319376 + }, + "statuses": { + "200": 32572, + "201": 3620, + "202": 2052, + "503": 4822 + } + }, + "successful_rps": 0.4999965896441693, + "error_ratio": 0.952, + "latency_success_ms": { + "0.5": 28.64320809021592, + "0.95": 42.32870787382126, + "0.99": 52.682749927043915 + }, + "statuses": { + "202": 120, + "503": 2380 + } + }, + { + "name": "db_recovery_2", + "start": 1794821183.300124, + "end": 1794821843.304569, + "before": { + "requests": 43088, + "success": 38265, + "errors": 4823, + "generator_dropped": 0, + "uploads_accepted": 2052, + "latency_success_ms": { + "0.5": 8.360208012163639, + "0.95": 17.539292108267546, + "0.99": 29.278708156198263 + }, + "statuses": { + "200": 32591, + "201": 3622, + "202": 2052, + "503": 4823 + } + }, + "after": { + "requests": 50018, + "success": 45195, + "errors": 4823, + "generator_dropped": 0, + "uploads_accepted": 2382, + "latency_success_ms": { + "0.5": 8.339041844010353, + "0.95": 17.473666928708553, + "0.99": 28.92820816487074 + }, + "statuses": { + "200": 38531, + "201": 4282, + "202": 2382, + "503": 4823 + } + }, + "successful_rps": 10.499933212878483, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.255416993051767, + "0.95": 17.043875064700842, + "0.99": 27.484582737088203 + }, + "statuses": { + "200": 5940, + "201": 660, + "202": 330 + } + }, + { + "name": "worker_short_3", + "start": 1794821843.637631, + "end": 1794821848.642217, + "before": { + "requests": 50021, + "success": 45198, + "errors": 4823, + "generator_dropped": 0, + "uploads_accepted": 2382, + "latency_success_ms": { + "0.5": 8.338292129337788, + "0.95": 17.473666928708553, + "0.99": 28.92820816487074 + }, + "statuses": { + "200": 38534, + "201": 4282, + "202": 2382, + "503": 4823 + } + }, + "after": { + "requests": 50074, + "success": 45251, + "errors": 4823, + "generator_dropped": 0, + "uploads_accepted": 2385, + "latency_success_ms": { + "0.5": 8.339041844010353, + "0.95": 17.478832975029945, + "0.99": 28.925917111337185 + }, + "statuses": { + "200": 38579, + "201": 4287, + "202": 2385, + "503": 4823 + } + }, + "successful_rps": 10.59029924161761, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.69549997150898, + "0.95": 24.411499965935946, + "0.99": 26.277707889676094 + }, + "statuses": { + "200": 45, + "201": 5, + "202": 3 + } + }, + { + "name": "worker_short_recovery_3", + "start": 1794821848.894581, + "end": 1794821908.89797, + "before": { + "requests": 50077, + "success": 45254, + "errors": 4823, + "generator_dropped": 0, + "uploads_accepted": 2385, + "latency_success_ms": { + "0.5": 8.338917046785355, + "0.95": 17.47949980199337, + "0.99": 28.92820816487074 + }, + "statuses": { + "200": 38581, + "201": 4288, + "202": 2385, + "503": 4823 + } + }, + "after": { + "requests": 50707, + "success": 45884, + "errors": 4823, + "generator_dropped": 0, + "uploads_accepted": 2415, + "latency_success_ms": { + "0.5": 8.349542040377855, + "0.95": 17.47399987652898, + "0.99": 28.896667063236237 + }, + "statuses": { + "200": 39121, + "201": 4348, + "202": 2415, + "503": 4823 + } + }, + "successful_rps": 10.499412402612851, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.833999745547771, + "0.95": 16.996500082314014, + "0.99": 23.976624943315983 + }, + "statuses": { + "200": 540, + "201": 60, + "202": 30 + } + }, + { + "name": "db_short_3", + "start": 1794821909.053349, + "end": 1794821914.0621068, + "before": { + "requests": 50708, + "success": 45885, + "errors": 4823, + "generator_dropped": 0, + "uploads_accepted": 2415, + "latency_success_ms": { + "0.5": 8.349542040377855, + "0.95": 17.47399987652898, + "0.99": 28.896667063236237 + }, + "statuses": { + "200": 39122, + "201": 4348, + "202": 2415, + "503": 4823 + } + }, + "after": { + "requests": 50761, + "success": 45908, + "errors": 4853, + "generator_dropped": 0, + "uploads_accepted": 2418, + "latency_success_ms": { + "0.5": 8.355749770998955, + "0.95": 17.529374919831753, + "0.99": 29.146083164960146 + }, + "statuses": { + "200": 39140, + "201": 4350, + "202": 2418, + "503": 4853 + } + }, + "successful_rps": 4.591960382015873, + "error_ratio": 0.5660377358490566, + "latency_success_ms": { + "0.5": 904.9617499113083, + "0.95": 1902.7691250666976, + "0.99": 2003.8470001891255 + }, + "statuses": { + "200": 18, + "201": 2, + "202": 3, + "503": 30 + } + }, + { + "name": "db_short_recovery_3", + "start": 1794821914.218946, + "end": 1794821974.227406, + "before": { + "requests": 50763, + "success": 45910, + "errors": 4853, + "generator_dropped": 0, + "uploads_accepted": 2418, + "latency_success_ms": { + "0.5": 8.355791680514812, + "0.95": 17.530541867017746, + "0.99": 29.146083164960146 + }, + "statuses": { + "200": 39142, + "201": 4350, + "202": 2418, + "503": 4853 + } + }, + "after": { + "requests": 51393, + "success": 46540, + "errors": 4853, + "generator_dropped": 0, + "uploads_accepted": 2448, + "latency_success_ms": { + "0.5": 8.35833279415965, + "0.95": 17.513417173177004, + "0.99": 29.061750043183565 + }, + "statuses": { + "200": 39682, + "201": 4410, + "202": 2448, + "503": 4853 + } + }, + "successful_rps": 10.498524622830507, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.560750167816877, + "0.95": 16.908416990190744, + "0.99": 24.552625138312578 + }, + "statuses": { + "200": 540, + "201": 60, + "202": 30 + } + }, + { + "name": "worker_long_3", + "start": 1794821974.6094139, + "end": 1794822214.61274, + "before": { + "requests": 51397, + "success": 46544, + "errors": 4853, + "generator_dropped": 0, + "uploads_accepted": 2448, + "latency_success_ms": { + "0.5": 8.358292281627655, + "0.95": 17.526624724268913, + "0.99": 29.09083291888237 + }, + "statuses": { + "200": 39686, + "201": 4410, + "202": 2448, + "503": 4853 + } + }, + "after": { + "requests": 53917, + "success": 49064, + "errors": 4853, + "generator_dropped": 0, + "uploads_accepted": 2568, + "latency_success_ms": { + "0.5": 8.352582808583975, + "0.95": 17.50358287245035, + "0.99": 28.98012474179268 + }, + "statuses": { + "200": 41846, + "201": 4650, + "202": 2568, + "503": 4853 + } + }, + "successful_rps": 10.499858507930577, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.27891705557704, + "0.95": 17.318916972726583, + "0.99": 25.771250016987324 + }, + "statuses": { + "200": 2160, + "201": 240, + "202": 120 + } + }, + { + "name": "worker_recovery_3", + "start": 1794822214.850919, + "end": 1794822874.8563461, + "before": { + "requests": 53920, + "success": 49067, + "errors": 4853, + "generator_dropped": 0, + "uploads_accepted": 2568, + "latency_success_ms": { + "0.5": 8.352875243872404, + "0.95": 17.50950003042817, + "0.99": 28.98012474179268 + }, + "statuses": { + "200": 41848, + "201": 4651, + "202": 2568, + "503": 4853 + } + }, + "after": { + "requests": 60850, + "success": 55997, + "errors": 4853, + "generator_dropped": 0, + "uploads_accepted": 2898, + "latency_success_ms": { + "0.5": 8.302957750856876, + "0.95": 17.357957549393177, + "0.99": 28.451249934732914 + }, + "statuses": { + "200": 47788, + "201": 5311, + "202": 2898, + "503": 4853 + } + }, + "successful_rps": 10.499917115518972, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.011457975953817, + "0.95": 16.36004215106368, + "0.99": 24.669541977345943 + }, + "statuses": { + "200": 5940, + "201": 660, + "202": 330 + } + }, + { + "name": "db_long_3", + "start": 1794822875.0330508, + "end": 1794823115.039806, + "before": { + "requests": 60851, + "success": 55998, + "errors": 4853, + "generator_dropped": 0, + "uploads_accepted": 2898, + "latency_success_ms": { + "0.5": 8.302582893520594, + "0.95": 17.357957549393177, + "0.99": 28.451249934732914 + }, + "statuses": { + "200": 47789, + "201": 5311, + "202": 2898, + "503": 4853 + } + }, + "after": { + "requests": 63351, + "success": 56118, + "errors": 7233, + "generator_dropped": 0, + "uploads_accepted": 3018, + "latency_success_ms": { + "0.5": 8.315250277519226, + "0.95": 17.595374956727028, + "0.99": 29.060583095997572 + }, + "statuses": { + "200": 47789, + "201": 5311, + "202": 3018, + "503": 7233 + } + }, + "successful_rps": 0.49998607674270545, + "error_ratio": 0.952, + "latency_success_ms": { + "0.5": 27.898625005036592, + "0.95": 41.145917028188705, + "0.99": 74.70795791596174 + }, + "statuses": { + "202": 120, + "503": 2380 + } + }, + { + "name": "db_recovery_3", + "start": 1794823115.229683, + "end": 1794823775.233116, + "before": { + "requests": 63373, + "success": 56139, + "errors": 7234, + "generator_dropped": 0, + "uploads_accepted": 3018, + "latency_success_ms": { + "0.5": 8.317166939377785, + "0.95": 17.63608306646347, + "0.99": 29.314542189240456 + }, + "statuses": { + "200": 47808, + "201": 5313, + "202": 3018, + "503": 7234 + } + }, + "after": { + "requests": 70303, + "success": 63069, + "errors": 7234, + "generator_dropped": 0, + "uploads_accepted": 3348, + "latency_success_ms": { + "0.5": 7.9916249960660934, + "0.95": 17.586166970431805, + "0.99": 29.451792128384113 + }, + "statuses": { + "200": 53748, + "201": 5973, + "202": 3348, + "503": 7234 + } + }, + "successful_rps": 10.499949325459296, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 6.970957852900028, + "0.95": 17.245042137801647, + "0.99": 32.710082828998566 + }, + "statuses": { + "200": 5940, + "201": 660, + "202": 330 + } + }, + { + "name": "tempo_unavailable", + "start": 1794823775.759885, + "end": 1794824075.766301, + "before": { + "requests": 70310, + "success": 63076, + "errors": 7234, + "generator_dropped": 0, + "uploads_accepted": 3349, + "latency_success_ms": { + "0.5": 7.991583086550236, + "0.95": 17.592458054423332, + "0.99": 29.46412516757846 + }, + "statuses": { + "200": 53753, + "201": 5974, + "202": 3349, + "503": 7234 + } + }, + "after": { + "requests": 73458, + "success": 66224, + "errors": 7234, + "generator_dropped": 0, + "uploads_accepted": 3498, + "latency_success_ms": { + "0.5": 7.856625132262707, + "0.95": 17.53687486052513, + "0.99": 29.35979189351201 + }, + "statuses": { + "200": 56453, + "201": 6273, + "202": 3498, + "503": 7234 + } + }, + "successful_rps": 10.499779013278001, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 6.962708197534084, + "0.95": 16.45737513899803, + "0.99": 25.757708121091127 + }, + "statuses": { + "200": 2700, + "201": 300, + "202": 150 + } + }, + { + "name": "tempo_recovery", + "start": 1794824076.114967, + "end": 1794824136.1225789, + "before": { + "requests": 73463, + "success": 66229, + "errors": 7234, + "generator_dropped": 0, + "uploads_accepted": 3499, + "latency_success_ms": { + "0.5": 7.856625132262707, + "0.95": 17.539499793201685, + "0.99": 29.365708120167255 + }, + "statuses": { + "200": 56456, + "201": 6274, + "202": 3499, + "503": 7234 + } + }, + "after": { + "requests": 74093, + "success": 66859, + "errors": 7234, + "generator_dropped": 0, + "uploads_accepted": 3529, + "latency_success_ms": { + "0.5": 7.835667114704847, + "0.95": 17.5331667996943, + "0.99": 29.314542189240456 + }, + "statuses": { + "200": 56996, + "201": 6334, + "202": 3529, + "503": 7234 + } + }, + "successful_rps": 10.49867386879547, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 7.084125187247992, + "0.95": 16.59925002604723, + "0.99": 26.641832664608955 + }, + "statuses": { + "200": 540, + "201": 60, + "202": 30 + } + }, + { + "name": "final_drain", + "start": 1794824136.201777, + "end": 1794824166.207861, + "before": { + "requests": 74094, + "success": 66860, + "errors": 7234, + "generator_dropped": 0, + "uploads_accepted": 3529, + "latency_success_ms": { + "0.5": 7.835667114704847, + "0.95": 17.5331667996943, + "0.99": 29.314542189240456 + }, + "statuses": { + "200": 56997, + "201": 6334, + "202": 3529, + "503": 7234 + } + }, + "after": { + "requests": 74094, + "success": 66860, + "errors": 7234, + "generator_dropped": 0, + "uploads_accepted": 3529, + "latency_success_ms": { + "0.5": 7.835667114704847, + "0.95": 17.5331667996943, + "0.99": 29.314542189240456 + }, + "statuses": { + "200": 56997, + "201": 6334, + "202": 3529, + "503": 7234 + } + }, + "successful_rps": 0.0, + "error_ratio": null, + "latency_success_ms": { + "0.5": null, + "0.95": null, + "0.99": null + }, + "statuses": {} + } + ], + "image_id": "sha256:d661ef49e464a482dcae899acf1513fa1cf57fae01664445e2dbfb323e65c28a", + "backend_before": { + "otelcol_exporter_sent_spans_total": [ + { + "metric": { + "__name__": "otelcol_exporter_sent_spans_total", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794823775.295, + "1298978" + ] + } + ], + "otelcol_exporter_send_failed_spans_total": [ + { + "metric": { + "__name__": "otelcol_exporter_send_failed_spans_total", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794823775.3, + "0" + ] + } + ], + "otelcol_exporter_queue_size": [ + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "logs", + "exporter": "otlphttp/loki", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794823775.305, + "0" + ] + }, + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "traces", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794823775.305, + "0" + ] + } + ], + "otelcol_receiver_accepted_spans_total": [ + { + "metric": { + "__name__": "otelcol_receiver_accepted_spans_total", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/receiver/receiverhelper", + "receiver": "otlp", + "transport": "http" + }, + "value": [ + 1794823775.311, + "1299025" + ] + } + ] + }, + "backend_after": { + "otelcol_exporter_sent_spans_total": [ + { + "metric": { + "__name__": "otelcol_exporter_sent_spans_total", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794824136.181, + "1311199" + ] + } + ], + "otelcol_exporter_send_failed_spans_total": [ + { + "metric": { + "__name__": "otelcol_exporter_send_failed_spans_total", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794824136.187, + "2524" + ] + } + ], + "otelcol_exporter_queue_size": [ + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "logs", + "exporter": "otlphttp/loki", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794824136.194, + "0" + ] + }, + { + "metric": { + "__name__": "otelcol_exporter_queue_size", + "data_type": "traces", + "exporter": "otlphttp/tempo", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/exporter/exporterhelper" + }, + "value": [ + 1794824136.194, + "0" + ] + } + ], + "otelcol_receiver_accepted_spans_total": [ + { + "metric": { + "__name__": "otelcol_receiver_accepted_spans_total", + "instance": "otel-collector:8888", + "job": "collector", + "otel_scope_name": "go.opentelemetry.io/collector/receiver/receiverhelper", + "receiver": "otlp", + "transport": "http" + }, + "value": [ + 1794824136.199, + "1349454" + ] + } + ] + }, + "accepted_ids_count": 3529, + "attachment_states": { + "ready": 3529 + }, + "message_persistence": { + "confirmed": 6334, + "stored": 6334, + "confirmed_missing": 0 + }, + "completed": true, + "finished": 1794824171.576171, + "traffic": { + "requests": 74094, + "success": 66860, + "errors": 7234, + "generator_dropped": 0, + "uploads_accepted": 3529, + "latency_success_ms": { + "0.5": 7.835667114704847, + "0.95": 17.5331667996943, + "0.99": 29.314542189240456 + }, + "statuses": { + "200": 56997, + "201": 6334, + "202": 3529, + "503": 7234 + } + } +} diff --git a/evidence/slo-run.txt b/evidence/slo-run.txt new file mode 100644 index 0000000..a577f21 --- /dev/null +++ b/evidence/slo-run.txt @@ -0,0 +1,151 @@ +baseline 900 seconds +baseline 2 s 17 requests +baseline 66 s 693 requests +baseline 131 s 1374 requests +baseline 195 s 2048 requests +baseline 260 s 2728 requests +baseline 324 s 3398 requests +baseline 388 s 4079 requests +baseline 453 s 4753 requests +baseline 517 s 5433 requests +baseline 582 s 6108 requests +baseline 646 s 6785 requests +baseline 711 s 7465 requests +baseline 775 s 8140 requests +baseline 840 s 8820 requests +worker_short_1 5 seconds +worker_short_1 1 s 9468 requests +worker_short_recovery_1 60 seconds +worker_short_recovery_1 2 s 9532 requests +db_short_1 5 seconds +db_short_1 2 s 10143 requests +db_short_recovery_1 60 seconds +db_short_recovery_1 2 s 10215 requests +worker_long_1 240 seconds +worker_long_1 2 s 10844 requests +worker_long_1 66 s 11525 requests +worker_long_1 131 s 12199 requests +worker_long_1 195 s 12876 requests +worker_recovery_1 660 seconds +worker_recovery_1 1 s 13365 requests +worker_recovery_1 66 s 14045 requests +worker_recovery_1 130 s 14719 requests +worker_recovery_1 195 s 15397 requests +worker_recovery_1 260 s 16078 requests +worker_recovery_1 324 s 16751 requests +worker_recovery_1 389 s 17433 requests +worker_recovery_1 452 s 18101 requests +worker_recovery_1 517 s 18782 requests +worker_recovery_1 581 s 19456 requests +worker_recovery_1 646 s 20136 requests +db_long_1 240 seconds +db_long_1 1 s 20283 requests +db_long_1 66 s 20957 requests +db_long_1 130 s 21630 requests +db_long_1 195 s 22307 requests +db_recovery_1 660 seconds +db_recovery_1 2 s 22827 requests +db_recovery_1 67 s 23508 requests +db_recovery_1 131 s 24181 requests +db_recovery_1 196 s 24863 requests +db_recovery_1 260 s 25535 requests +db_recovery_1 325 s 26214 requests +db_recovery_1 390 s 26895 requests +db_recovery_1 454 s 27567 requests +db_recovery_1 518 s 28248 requests +db_recovery_1 582 s 28917 requests +db_recovery_1 647 s 29599 requests +worker_short_2 5 seconds +worker_short_2 2 s 29760 requests +worker_short_recovery_2 60 seconds +worker_short_recovery_2 2 s 29816 requests +db_short_2 5 seconds +db_short_2 2 s 30427 requests +db_short_recovery_2 60 seconds +db_short_recovery_2 2 s 30502 requests +worker_long_2 240 seconds +worker_long_2 2 s 31135 requests +worker_long_2 67 s 31817 requests +worker_long_2 131 s 32488 requests +worker_long_2 196 s 33170 requests +worker_recovery_2 660 seconds +worker_recovery_2 2 s 33658 requests +worker_recovery_2 67 s 34341 requests +worker_recovery_2 131 s 35009 requests +worker_recovery_2 196 s 35690 requests +worker_recovery_2 260 s 36363 requests +worker_recovery_2 325 s 37044 requests +worker_recovery_2 389 s 37718 requests +worker_recovery_2 453 s 38395 requests +worker_recovery_2 518 s 39077 requests +worker_recovery_2 582 s 39749 requests +worker_recovery_2 647 s 40431 requests +db_long_2 240 seconds +db_long_2 2 s 40569 requests +db_long_2 67 s 41251 requests +db_long_2 131 s 41920 requests +db_long_2 196 s 42601 requests +db_recovery_2 660 seconds +db_recovery_2 2 s 43111 requests +db_recovery_2 67 s 43793 requests +db_recovery_2 132 s 44475 requests +db_recovery_2 197 s 45157 requests +db_recovery_2 261 s 45825 requests +db_recovery_2 326 s 46506 requests +db_recovery_2 390 s 47180 requests +db_recovery_2 455 s 47862 requests +db_recovery_2 519 s 48535 requests +db_recovery_2 583 s 49212 requests +db_recovery_2 648 s 49894 requests +worker_short_3 5 seconds +worker_short_3 2 s 50045 requests +worker_short_recovery_3 60 seconds +worker_short_recovery_3 2 s 50100 requests +db_short_3 5 seconds +db_short_3 2 s 50712 requests +db_short_recovery_3 60 seconds +db_short_recovery_3 2 s 50786 requests +worker_long_3 240 seconds +worker_long_3 2 s 51420 requests +worker_long_3 67 s 52103 requests +worker_long_3 132 s 52782 requests +worker_long_3 197 s 53464 requests +worker_recovery_3 660 seconds +worker_recovery_3 2 s 53943 requests +worker_recovery_3 67 s 54626 requests +worker_recovery_3 132 s 55302 requests +worker_recovery_3 197 s 55984 requests +worker_recovery_3 261 s 56656 requests +worker_recovery_3 326 s 57339 requests +worker_recovery_3 389 s 58007 requests +worker_recovery_3 454 s 58689 requests +worker_recovery_3 518 s 59361 requests +worker_recovery_3 583 s 60043 requests +worker_recovery_3 647 s 60716 requests +db_long_3 240 seconds +db_long_3 2 s 60854 requests +db_long_3 67 s 61536 requests +db_long_3 132 s 62213 requests +db_long_3 197 s 62896 requests +db_recovery_3 660 seconds +db_recovery_3 2 s 63396 requests +db_recovery_3 60 s 64004 requests +db_recovery_3 124 s 64680 requests +db_recovery_3 185 s 65319 requests +db_recovery_3 249 s 65991 requests +db_recovery_3 309 s 66622 requests +db_recovery_3 374 s 67303 requests +db_recovery_3 435 s 67938 requests +db_recovery_3 499 s 68615 requests +db_recovery_3 559 s 69246 requests +db_recovery_3 624 s 69926 requests +tempo_unavailable 300 seconds +tempo_unavailable 2 s 70333 requests +tempo_unavailable 60 s 70941 requests +tempo_unavailable 124 s 71613 requests +tempo_unavailable 184 s 72245 requests +tempo_unavailable 249 s 72921 requests +tempo_recovery 60 seconds +tempo_recovery 2 s 73486 requests +final_drain 30 seconds +final_drain 2 s 74094 requests diff --git a/evidence/slo-summary-stdout.txt b/evidence/slo-summary-stdout.txt new file mode 100644 index 0000000..847a446 --- /dev/null +++ b/evidence/slo-summary-stdout.txt @@ -0,0 +1,342 @@ +{ + "completed": true, + "duration_seconds": 7092.821509838104, + "parameters": { + "warmup": 900, + "long": 240, + "cooldown": 660, + "repeats": 3, + "backend": 300, + "output": "evidence/slo-experiment.json" + }, + "incidents": [ + { + "phase": "worker_short_1", + "duration": 5.004392862319946, + "alerts": { + "QueueLag": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ReadyBurnRateFast": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ReadyBurnRateSlow": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + } + }, + "successful_rps": 10.3908971041871, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 7.937416899949312, + "0.95": 16.20679209008813, + "0.99": 17.71804178133607 + } + }, + { + "phase": "db_short_1", + "duration": 5.0065529346466064, + "alerts": { + "ApiBurnRateFast": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ApiBurnRateSlow": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + } + }, + "successful_rps": 0.5992161038489773, + "error_ratio": 0.9090909090909091, + "latency_success_ms": { + "0.5": 23.7681670114398, + "0.95": 27.764666825532913, + "0.99": 27.764666825532913 + } + }, + { + "phase": "worker_long_1", + "duration": 240.00321888923645, + "alerts": { + "QueueLag": { + "detection_seconds": 64.16309976577759, + "resolved_after_restore_seconds": 16.647381067276, + "fired": true + }, + "ReadyBurnRateFast": { + "detection_seconds": 81.4437928199768, + "resolved_after_restore_seconds": 64.19608902931213, + "fired": true + }, + "ReadyBurnRateSlow": { + "detection_seconds": 91.5375108718872, + "resolved_after_restore_seconds": 123.94137406349182, + "fired": true + } + }, + "successful_rps": 10.499865558974752, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.786541875451803, + "0.95": 17.55095785483718, + "0.99": 28.451249934732914 + } + }, + { + "phase": "db_long_1", + "duration": 240.002925157547, + "alerts": { + "ApiBurnRateFast": { + "detection_seconds": 53.87334394454956, + "resolved_after_restore_seconds": 64.97496175765991, + "fired": true + }, + "ApiBurnRateSlow": { + "detection_seconds": 63.97737908363342, + "resolved_after_restore_seconds": 124.8059549331665, + "fired": true + } + }, + "successful_rps": 0.49999409771972575, + "error_ratio": 0.952, + "latency_success_ms": { + "0.5": 27.061082888394594, + "0.95": 41.25562496483326, + "0.99": 48.944166861474514 + } + }, + { + "phase": "worker_short_2", + "duration": 5.0105671882629395, + "alerts": { + "QueueLag": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ReadyBurnRateFast": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ReadyBurnRateSlow": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + } + }, + "successful_rps": 10.37807944264453, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 6.861874833703041, + "0.95": 17.471542116254568, + "0.99": 27.467207983136177 + } + }, + { + "phase": "db_short_2", + "duration": 5.0033698081970215, + "alerts": { + "ApiBurnRateFast": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ApiBurnRateSlow": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + } + }, + "successful_rps": 0.5995964668020037, + "error_ratio": 0.9090909090909091, + "latency_success_ms": { + "0.5": 31.136707868427038, + "0.95": 34.025250002741814, + "0.99": 34.025250002741814 + } + }, + { + "phase": "worker_long_2", + "duration": 240.0071098804474, + "alerts": { + "QueueLag": { + "detection_seconds": 64.87712001800537, + "resolved_after_restore_seconds": 17.59833526611328, + "fired": true + }, + "ReadyBurnRateFast": { + "detection_seconds": 77.18091893196106, + "resolved_after_restore_seconds": 72.45875120162964, + "fired": true + }, + "ReadyBurnRateSlow": { + "detection_seconds": 87.29317092895508, + "resolved_after_restore_seconds": 129.3502321243286, + "fired": true + } + }, + "successful_rps": 10.499693843708734, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.250332903116941, + "0.95": 16.793041955679655, + "0.99": 24.823333136737347 + } + }, + { + "phase": "db_long_2", + "duration": 240.00172305107117, + "alerts": { + "ApiBurnRateFast": { + "detection_seconds": 49.7854859828949, + "resolved_after_restore_seconds": 60.06502294540405, + "fired": true + }, + "ApiBurnRateSlow": { + "detection_seconds": 59.878856897354126, + "resolved_after_restore_seconds": 119.95286202430725, + "fired": true + } + }, + "successful_rps": 0.4999965896441693, + "error_ratio": 0.952, + "latency_success_ms": { + "0.5": 28.64320809021592, + "0.95": 42.32870787382126, + "0.99": 52.682749927043915 + } + }, + { + "phase": "worker_short_3", + "duration": 5.0045859813690186, + "alerts": { + "QueueLag": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ReadyBurnRateFast": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ReadyBurnRateSlow": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + } + }, + "successful_rps": 10.59029924161761, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.69549997150898, + "0.95": 24.411499965935946, + "0.99": 26.277707889676094 + } + }, + { + "phase": "db_short_3", + "duration": 5.008757829666138, + "alerts": { + "ApiBurnRateFast": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ApiBurnRateSlow": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + } + }, + "successful_rps": 4.591960382015873, + "error_ratio": 0.5660377358490566, + "latency_success_ms": { + "0.5": 904.9617499113083, + "0.95": 1902.7691250666976, + "0.99": 2003.8470001891255 + } + }, + { + "phase": "worker_long_3", + "duration": 240.00332617759705, + "alerts": { + "QueueLag": { + "detection_seconds": 59.91160011291504, + "resolved_after_restore_seconds": 12.543959856033325, + "fired": true + }, + "ReadyBurnRateFast": { + "detection_seconds": 72.20107531547546, + "resolved_after_restore_seconds": 65.21866607666016, + "fired": true + }, + "ReadyBurnRateSlow": { + "detection_seconds": 87.36326217651367, + "resolved_after_restore_seconds": 124.68797492980957, + "fired": true + } + }, + "successful_rps": 10.499858507930577, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.27891705557704, + "0.95": 17.318916972726583, + "0.99": 25.771250016987324 + } + }, + { + "phase": "db_long_3", + "duration": 240.00675511360168, + "alerts": { + "ApiBurnRateFast": { + "detection_seconds": 44.78353834152222, + "resolved_after_restore_seconds": 55.135868310928345, + "fired": true + }, + "ApiBurnRateSlow": { + "detection_seconds": 54.89339232444763, + "resolved_after_restore_seconds": 119.52835726737976, + "fired": true + } + }, + "successful_rps": 0.49998607674270545, + "error_ratio": 0.952, + "latency_success_ms": { + "0.5": 27.898625005036592, + "0.95": 41.145917028188705, + "0.99": 74.70795791596174 + } + } + ], + "traffic": { + "requests": 74094, + "success": 66860, + "errors": 7234, + "generator_dropped": 0, + "uploads_accepted": 3529, + "latency_success_ms": { + "0.5": 7.835667114704847, + "0.95": 17.5331667996943, + "0.99": 29.314542189240456 + }, + "statuses": { + "200": 56997, + "201": 6334, + "202": 3529, + "503": 7234 + } + }, + "attachment_states": { + "ready": 3529 + }, + "image_id": "sha256:d661ef49e464a482dcae899acf1513fa1cf57fae01664445e2dbfb323e65c28a" +} diff --git a/evidence/slo-summary.json b/evidence/slo-summary.json new file mode 100644 index 0000000..847a446 --- /dev/null +++ b/evidence/slo-summary.json @@ -0,0 +1,342 @@ +{ + "completed": true, + "duration_seconds": 7092.821509838104, + "parameters": { + "warmup": 900, + "long": 240, + "cooldown": 660, + "repeats": 3, + "backend": 300, + "output": "evidence/slo-experiment.json" + }, + "incidents": [ + { + "phase": "worker_short_1", + "duration": 5.004392862319946, + "alerts": { + "QueueLag": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ReadyBurnRateFast": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ReadyBurnRateSlow": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + } + }, + "successful_rps": 10.3908971041871, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 7.937416899949312, + "0.95": 16.20679209008813, + "0.99": 17.71804178133607 + } + }, + { + "phase": "db_short_1", + "duration": 5.0065529346466064, + "alerts": { + "ApiBurnRateFast": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ApiBurnRateSlow": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + } + }, + "successful_rps": 0.5992161038489773, + "error_ratio": 0.9090909090909091, + "latency_success_ms": { + "0.5": 23.7681670114398, + "0.95": 27.764666825532913, + "0.99": 27.764666825532913 + } + }, + { + "phase": "worker_long_1", + "duration": 240.00321888923645, + "alerts": { + "QueueLag": { + "detection_seconds": 64.16309976577759, + "resolved_after_restore_seconds": 16.647381067276, + "fired": true + }, + "ReadyBurnRateFast": { + "detection_seconds": 81.4437928199768, + "resolved_after_restore_seconds": 64.19608902931213, + "fired": true + }, + "ReadyBurnRateSlow": { + "detection_seconds": 91.5375108718872, + "resolved_after_restore_seconds": 123.94137406349182, + "fired": true + } + }, + "successful_rps": 10.499865558974752, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.786541875451803, + "0.95": 17.55095785483718, + "0.99": 28.451249934732914 + } + }, + { + "phase": "db_long_1", + "duration": 240.002925157547, + "alerts": { + "ApiBurnRateFast": { + "detection_seconds": 53.87334394454956, + "resolved_after_restore_seconds": 64.97496175765991, + "fired": true + }, + "ApiBurnRateSlow": { + "detection_seconds": 63.97737908363342, + "resolved_after_restore_seconds": 124.8059549331665, + "fired": true + } + }, + "successful_rps": 0.49999409771972575, + "error_ratio": 0.952, + "latency_success_ms": { + "0.5": 27.061082888394594, + "0.95": 41.25562496483326, + "0.99": 48.944166861474514 + } + }, + { + "phase": "worker_short_2", + "duration": 5.0105671882629395, + "alerts": { + "QueueLag": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ReadyBurnRateFast": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ReadyBurnRateSlow": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + } + }, + "successful_rps": 10.37807944264453, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 6.861874833703041, + "0.95": 17.471542116254568, + "0.99": 27.467207983136177 + } + }, + { + "phase": "db_short_2", + "duration": 5.0033698081970215, + "alerts": { + "ApiBurnRateFast": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ApiBurnRateSlow": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + } + }, + "successful_rps": 0.5995964668020037, + "error_ratio": 0.9090909090909091, + "latency_success_ms": { + "0.5": 31.136707868427038, + "0.95": 34.025250002741814, + "0.99": 34.025250002741814 + } + }, + { + "phase": "worker_long_2", + "duration": 240.0071098804474, + "alerts": { + "QueueLag": { + "detection_seconds": 64.87712001800537, + "resolved_after_restore_seconds": 17.59833526611328, + "fired": true + }, + "ReadyBurnRateFast": { + "detection_seconds": 77.18091893196106, + "resolved_after_restore_seconds": 72.45875120162964, + "fired": true + }, + "ReadyBurnRateSlow": { + "detection_seconds": 87.29317092895508, + "resolved_after_restore_seconds": 129.3502321243286, + "fired": true + } + }, + "successful_rps": 10.499693843708734, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.250332903116941, + "0.95": 16.793041955679655, + "0.99": 24.823333136737347 + } + }, + { + "phase": "db_long_2", + "duration": 240.00172305107117, + "alerts": { + "ApiBurnRateFast": { + "detection_seconds": 49.7854859828949, + "resolved_after_restore_seconds": 60.06502294540405, + "fired": true + }, + "ApiBurnRateSlow": { + "detection_seconds": 59.878856897354126, + "resolved_after_restore_seconds": 119.95286202430725, + "fired": true + } + }, + "successful_rps": 0.4999965896441693, + "error_ratio": 0.952, + "latency_success_ms": { + "0.5": 28.64320809021592, + "0.95": 42.32870787382126, + "0.99": 52.682749927043915 + } + }, + { + "phase": "worker_short_3", + "duration": 5.0045859813690186, + "alerts": { + "QueueLag": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ReadyBurnRateFast": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ReadyBurnRateSlow": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + } + }, + "successful_rps": 10.59029924161761, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.69549997150898, + "0.95": 24.411499965935946, + "0.99": 26.277707889676094 + } + }, + { + "phase": "db_short_3", + "duration": 5.008757829666138, + "alerts": { + "ApiBurnRateFast": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + }, + "ApiBurnRateSlow": { + "detection_seconds": null, + "resolved_after_restore_seconds": null, + "fired": false + } + }, + "successful_rps": 4.591960382015873, + "error_ratio": 0.5660377358490566, + "latency_success_ms": { + "0.5": 904.9617499113083, + "0.95": 1902.7691250666976, + "0.99": 2003.8470001891255 + } + }, + { + "phase": "worker_long_3", + "duration": 240.00332617759705, + "alerts": { + "QueueLag": { + "detection_seconds": 59.91160011291504, + "resolved_after_restore_seconds": 12.543959856033325, + "fired": true + }, + "ReadyBurnRateFast": { + "detection_seconds": 72.20107531547546, + "resolved_after_restore_seconds": 65.21866607666016, + "fired": true + }, + "ReadyBurnRateSlow": { + "detection_seconds": 87.36326217651367, + "resolved_after_restore_seconds": 124.68797492980957, + "fired": true + } + }, + "successful_rps": 10.499858507930577, + "error_ratio": 0.0, + "latency_success_ms": { + "0.5": 8.27891705557704, + "0.95": 17.318916972726583, + "0.99": 25.771250016987324 + } + }, + { + "phase": "db_long_3", + "duration": 240.00675511360168, + "alerts": { + "ApiBurnRateFast": { + "detection_seconds": 44.78353834152222, + "resolved_after_restore_seconds": 55.135868310928345, + "fired": true + }, + "ApiBurnRateSlow": { + "detection_seconds": 54.89339232444763, + "resolved_after_restore_seconds": 119.52835726737976, + "fired": true + } + }, + "successful_rps": 0.49998607674270545, + "error_ratio": 0.952, + "latency_success_ms": { + "0.5": 27.898625005036592, + "0.95": 41.145917028188705, + "0.99": 74.70795791596174 + } + } + ], + "traffic": { + "requests": 74094, + "success": 66860, + "errors": 7234, + "generator_dropped": 0, + "uploads_accepted": 3529, + "latency_success_ms": { + "0.5": 7.835667114704847, + "0.95": 17.5331667996943, + "0.99": 29.314542189240456 + }, + "statuses": { + "200": 56997, + "201": 6334, + "202": 3529, + "503": 7234 + } + }, + "attachment_states": { + "ready": 3529 + }, + "image_id": "sha256:d661ef49e464a482dcae899acf1513fa1cf57fae01664445e2dbfb323e65c28a" +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..dca3c20 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:18086" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/storage-checks.txt b/evidence/storage-checks.txt new file mode 100644 index 0000000..87252cb --- /dev/null +++ b/evidence/storage-checks.txt @@ -0,0 +1,109 @@ +./scripts/test-storage.sh +=== RUN TestIdempotencyTransactionAndExpiry +2026/11/16 07:14:23 INFO connected to database db=lab5_atomic_1794813262_55454 +2026/11/16 07:14:23 INFO successfully initialized migrator +2026/11/16 07:14:23 INFO successfully loaded migrations error= +2026/11/16 07:14:23 INFO postgres_pool_config max_conns=8 connect_timeout=500ms statement_timeout_ms=2000 +2026/11/16 07:14:23 INFO successfully connected to postgres pool +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 WARN chat_cache_read_error chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 error="cache disabled for storage test" +2026/11/16 07:14:23 INFO chat_db_lookup chat_id=0296a27b-f827-49f0-b6ba-a7793543ed90 +=== RUN TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock +=== NAME TestIdempotencyTransactionAndExpiry + idempotency_test.go:295: rollback removed both side effect and key; expired key created one new message +--- PASS: TestIdempotencyTransactionAndExpiry (0.11s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules (0.01s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8 (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock (0.05s) +PASS diff --git a/evidence/storage.json b/evidence/storage.json new file mode 100644 index 0000000..681de15 --- /dev/null +++ b/evidence/storage.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "scenario": "queued_staging_protected", + "result": "PASS", + "attachment_id": "34aa2186-8c54-49ff-b517-03af038fc35c" + }, + { + "scenario": "staging_deleted_after_ready", + "result": "PASS", + "attachment_id": "34aa2186-8c54-49ff-b517-03af038fc35c" + }, + { + "scenario": "failed_retention_and_missing_source_replay", + "result": "PASS", + "age_simulated_hours": 25 + }, + { + "scenario": "verified_restore_then_replay", + "result": "PASS", + "job_id": "c425d43e-a61c-4501-b900-8fb75fe0add1" + } + ] +} diff --git a/evidence/telemetry-images.json b/evidence/telemetry-images.json new file mode 100644 index 0000000..9fd5082 --- /dev/null +++ b/evidence/telemetry-images.json @@ -0,0 +1,37 @@ +[ + { + "reference": "otel/opentelemetry-collector-contrib:0.140.0", + "id": "sha256:ffc818ee108d0b934fd14207fd87ff247c9b64f344ed349d0b66c166c18a2312", + "repo_digests": [ + "otel/opentelemetry-collector-contrib@sha256:ffc818ee108d0b934fd14207fd87ff247c9b64f344ed349d0b66c166c18a2312" + ] + }, + { + "reference": "prom/prometheus:v3.16.0", + "id": "sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06", + "repo_digests": [ + "prom/prometheus@sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06" + ] + }, + { + "reference": "grafana/loki:3.7.10", + "id": "sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7", + "repo_digests": [ + "grafana/loki@sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7" + ] + }, + { + "reference": "grafana/tempo:2.9.0", + "id": "sha256:65a5789759435f1ef696f1953258b9bbdb18eb571d5ce711ff812d2e128288a4", + "repo_digests": [ + "grafana/tempo@sha256:65a5789759435f1ef696f1953258b9bbdb18eb571d5ce711ff812d2e128288a4" + ] + }, + { + "reference": "grafana/grafana:13.3.1", + "id": "sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf", + "repo_digests": [ + "grafana/grafana@sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf" + ] + } +] diff --git a/infra/README.md b/infra/README.md index 5a39e86..9980764 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,7 @@ -# Инфраструктура студента +# Стенд lab6 -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Основная топология — `compose.yaml`; запуск `make up`, обычная остановка `make down` сохраняет volumes. Bootstrap создаёт отдельные синтетические credentials в игнорируемых `.env.lab6` и `secrets/lab6`, не меняя существующие значения. Источники приложения собираются из `service/`. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +HAProxy публикует только `127.0.0.1:18086`; Grafana — `127.0.0.1:13006`. PostgreSQL, Redis, S3, RabbitMQ, Collector и хранилища telemetry не публикуют порты на host. API-реплик две; приватный S3 находится в отдельной internal storage network. Проект и volumes не совпадают со стендом lab5. -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Роли, версии, health/readiness, лимиты и переменные описаны в [отчёте](../REPORT.md). Конфигурация [наблюдаемости](otel/README.md), [runbook](otel/RUNBOOK.md). Конфигурация ImageMagick ограничивает ресурсы преобразования изображений. `compose.faults.yaml` включает только явно запускаемые учебные барьеры; обычный запуск их не включает. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..b0626d6 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,39 @@ +global + maxconn 4096 + stats socket /tmp/haproxy.sock mode 600 level admin + stats socket ipv4@127.0.0.1:9999 level admin + +defaults + mode http + timeout connect 500ms + timeout client 125s + timeout server 125s + timeout check 500ms + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + +frontend public + bind :8080 + http-request deny deny_status 404 if { path_beg /internal } + default_backend api + +backend api + balance roundrobin + http-reuse always + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + server-template replica 1-4 api:8080 check inter 1s fall 2 rise 1 resolvers docker init-addr none resolve-prefer ipv4 + +listen statistics + bind :8404 + stats enable + stats uri /stats diff --git a/infra/imagemagick-policy.xml b/infra/imagemagick-policy.xml new file mode 100644 index 0000000..decc717 --- /dev/null +++ b/infra/imagemagick-policy.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/infra/otel/README.md b/infra/otel/README.md index c46fe89..b0b2759 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -1,7 +1,20 @@ -# Конфигурация наблюдаемости — реализовать +# Наблюдаемость lab6 -Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. +API / worker / dispatcher → OTLP HTTP/protobuf :4318 → отдельный Collector. -Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. +- Metrics: Collector Prometheus exporter :8889 → Prometheus, TSDB volume, 7 дней / 1 GiB. +- Logs: Collector `otlphttp/loki` → Loki native `/otlp`, TSDB v13, filesystem volume, retention 7 дней. +- Traces: Collector `otlphttp/tempo` → Tempo OTLP :4318, WAL + local blocks volume, retention 7 дней. +- Grafana хранит UI-настройки в собственном volume; datasource/dashboard provisioning находится здесь в Git. -Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. +Административные/OTLP порты не публикуются на host; Grafana доступна только через localhost:13006. Образы закреплены явными версиями в Compose; фактические image IDs сохраняет `scripts/capture-environment.py`. + +Collector: memory limiter: hard limit 180 MiB, soft limit 140 MiB (spike 40 MiB), контейнер 256 MiB; batch до 256 элементов, timeout 1s; по 32 запроса на exporter и два consumers, retry до 30s. SDK traces/logs: очереди по 1024 элементов, batch 128, timeout экспорта 1s, без блокирования бизнес-запроса; метрики экспортируются раз в 5s. RAM и количество consumers конечны; очереди telemetry не durable. + +Имена стандартных атрибутов выбраны по Semantic Conventions 1.37.0; собственные имена инструментов и их соответствие контракту перечислены в REPORT.md. + +Resource attributes: service.name, service.version=lab6, service.instance.id и deployment.environment.name=lab. Реплика использует один ID и для OTel resource, и для X-Instance-Id. Default trace sampler — ParentBased(AlwaysSample), то есть 100% новых корней; входящий sampled flag сохраняется. Sampling не применяется к метрикам. + +Политика данных: SQL-параметры, HTTP body/Authorization/URL и object keys не инструментируются. Логгер заменяет исходные error values безопасной категорией. Технические job/request IDs допустимы только в логах; labels метрик содержат route template, method/status class, job type/outcome, имя очереди и resource attributes. Уровень info, шумные cache-hit/miss события оставлены на debug. + +[Runbook и SLO](RUNBOOK.md), [отчёт](../../REPORT.md), [dashboard JSON](dashboards/messenger.json). diff --git a/infra/otel/RUNBOOK.md b/infra/otel/RUNBOOK.md new file mode 100644 index 0000000..5c2bcb6 --- /dev/null +++ b/infra/otel/RUNBOOK.md @@ -0,0 +1,48 @@ +# Диагностика lab6 + +Открыть Grafana: http://127.0.0.1:13006/d/messenger-lab6, пользователь `admin`. +Локальный пароль находится в `secrets/lab6/grafana_password`; не прикладывать его к отчёту. +Dashboard, datasource и alerts загружаются из Git. Внешняя доставка уведомлений не настроена. + +## Порядок действий + +1. Выбрать интервал инцидента. Проверить свежесть `messenger_observation_timestamp`, состояние scrape `up` и ошибки Collector. Пропуск данных не означает нулевую очередь или отсутствие ошибок. +2. Сопоставить HTTP RPS/5xx и p95 с backlog, возрастом jobs, worker busy/capacity и DB pool. Процентили вычисляются из суммарных buckets, а не усредняются между репликами. +3. В Explore → Tempo найти trace по времени/service.name; затем открыть связанные логи. Для конкретного upload взять trace ID из `http_request` или `job_received`. В Loki фильтр: `{service_name=~"messenger-.+"} | trace_id="TRACE_ID"`. +4. Сформулировать причину по сигналам. После этого сверить её с журналом воздействия. Проверить восстановление записи, выход jobs в ready, снижение backlog и исчезновение firing alerts. + +## Worker остановлен + +Признаки: растут `messenger_queue_depth`, `messenger_jobs_pending` и `messenger_queue_oldest_age_seconds`; HTTP upload продолжает возвращать 202. Свежие `job_received`/`job_ready` отсутствуют, новые consumer spans не появляются. Последняя worker gauge может оставаться в Prometheus некоторое время: нулевой busy сам по себе не доказывает остановку процесса. + +Alerts: `QueueLag`, затем `ReadyBurnRateFast`/`ReadyBurnRateSlow`. Даже незавершённые jobs учитываются как bad после 30 секунд: оценку выполняет dispatcher. + +Восстановление: `./scripts/compose.sh start worker`. Проверить новые consumer spans, ready, опустошение очереди. Ready-SLO уже нарушенных jobs остаётся bad после восстановления; burn-rate снижается только по мере выхода ошибок из окон. + +## PostgreSQL замедлена + +Признаки: HTTP 5xx и histogram duration растут; в trace длительность находится внутри `postgresql SELECT/INSERT`, заканчивается безопасным error status. `messenger_db_locks_waiting` подтверждает ожидание DB lock. Pool used/waiting помогает отличить ожидание соединения от выполняющегося запроса. Upload и worker могут работать, если заблокирована только таблица messages. Это не общий отказ PostgreSQL и не задержка преобразования изображения. + +Alerts: `ApiBurnRateFast`/`ApiBurnRateSlow`. В эксперименте блокировка имеет ограничение времени и отдельный `application_name=lab6_slo_injection`. Скрипт всегда завершает эту DB-сессию в finally. При аварийном завершении внешнего процесса оператор может выполнить: + +```sh +./scripts/compose.sh exec -T postgres psql -U messenger -d messenger -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name='lab6_slo_injection'" +``` + +Затем проверить успешные запросы и завершение alerts. SELECT 1 не блокируется таблицей messages, но исчерпание пула может временно нарушить readiness и добавить ответы LB. Учебный профиль использует 16 соединений на процесс, чтобы при данной нагрузке сохранить запас для health. Ответы самого LB всё равно учитываются генератором отдельно. + +## Telemetry backend недоступен + +Признаки: растут `otelcol_exporter_send_failed_spans_total`, появляются retry и bounded queue; после исчерпания бюджета растёт разница accepted/sent spans. Сопоставить `otelcol_exporter_queue_size` с capacity и RSS Collector. CPU/RAM процесса приложения доступны через оставшийся metrics pipeline. + +Восстановление: `./scripts/compose.sh start tempo`. Потерянные spans не восстанавливаются из бизнес-БД; это допустимая потеря telemetry, не потеря сообщений. Проверить новые traces и опустошение exporter queue. Ошибки отправки включают повторные попытки, поэтому их число нельзя приравнивать числу потерянных spans. + +## SLI и правила + +Учебное окно SLO — 1 час; бюджет ошибок — 1%. HTTP good = ответ бизнес-API без 5xx, denominator включает все `/api/v1/*`, в том числе 4xx. Health/internal исключены. Поэтому это серверный SLI, а не полный клиентский availability SLI: сетевые отказы и ответы самого LB отдельно учитываются генератором. + +Ready good = durable job первой обработки/новой версии достигла ready за ≤30 секунд от записанного accepted_at; terminal failure или deadline без ready = bad. Replay сохраняет первый SLO-результат. accepted_at фиксируется в транзакции durable handoff непосредственно перед COMMIT; небольшой интервал до подтверждения COMMIT входит в измерение. ready_at также фиксируется в транзакции публикации результата. Это серверные timestamps, не время полного скачивания клиентом. + +`error_ratio = bad / total`, `burn_rate = error_ratio / 0.01`. Fast: оба окна 1m и 5m >6×, for=20s. Slow: оба окна 2m и 10m >3×, for=30s. Минимум событий в коротком окне: API 20, ready 5. При отсутствии событий ratio не определена; отсутствие firing при малом трафике не доказывает SLO. + +Окна намеренно подходят для часового учебного стенда; это не стандартный 30-дневный production SLO. Scrape/evaluation/export — 5s, observer — примерно 1s. Регистрация результата во время отказа DB задерживается до восстановления; `ObservationsStale` показывает этот пробел. diff --git a/infra/otel/alerts.yaml b/infra/otel/alerts.yaml new file mode 100644 index 0000000..686b412 --- /dev/null +++ b/infra/otel/alerts.yaml @@ -0,0 +1,126 @@ +{ + "groups": [ + { + "name": "messenger-slo", + "interval": "5s", + "rules": [ + { + "record": "messenger:api_error_ratio:1m", + "expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[1m])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[1m]))" + }, + { + "record": "messenger:api_error_ratio:2m", + "expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[2m])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[2m]))" + }, + { + "record": "messenger:api_error_ratio:5m", + "expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[5m])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[5m]))" + }, + { + "record": "messenger:api_error_ratio:10m", + "expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[10m])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[10m]))" + }, + { + "alert": "ApiBurnRateFast", + "expr": "(messenger:api_error_ratio:1m > 0.06) and (messenger:api_error_ratio:5m > 0.06) and (sum(increase(messenger_http_requests_total{route=~\"/api/v1/.*\"}[1m])) >= 20)", + "for": "20s", + "labels": { + "severity": "warning", + "slo": "api" + }, + "annotations": { + "summary": "api SLO budget burning above 6x", + "runbook": "infra/otel/RUNBOOK.md" + } + }, + { + "alert": "ApiBurnRateSlow", + "expr": "(messenger:api_error_ratio:2m > 0.03) and (messenger:api_error_ratio:10m > 0.03) and (sum(increase(messenger_http_requests_total{route=~\"/api/v1/.*\"}[2m])) >= 20)", + "for": "30s", + "labels": { + "severity": "warning", + "slo": "api" + }, + "annotations": { + "summary": "api SLO budget burning above 3x", + "runbook": "infra/otel/RUNBOOK.md" + } + }, + { + "record": "messenger:ready_error_ratio:1m", + "expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[1m])) / sum(rate(messenger_ready_slo_total[1m]))" + }, + { + "record": "messenger:ready_error_ratio:2m", + "expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[2m])) / sum(rate(messenger_ready_slo_total[2m]))" + }, + { + "record": "messenger:ready_error_ratio:5m", + "expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[5m])) / sum(rate(messenger_ready_slo_total[5m]))" + }, + { + "record": "messenger:ready_error_ratio:10m", + "expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[10m])) / sum(rate(messenger_ready_slo_total[10m]))" + }, + { + "alert": "ReadyBurnRateFast", + "expr": "(messenger:ready_error_ratio:1m > 0.06) and (messenger:ready_error_ratio:5m > 0.06) and (sum(increase(messenger_ready_slo_total[1m])) >= 5)", + "for": "20s", + "labels": { + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "summary": "ready SLO budget burning above 6x", + "runbook": "infra/otel/RUNBOOK.md" + } + }, + { + "alert": "ReadyBurnRateSlow", + "expr": "(messenger:ready_error_ratio:2m > 0.03) and (messenger:ready_error_ratio:10m > 0.03) and (sum(increase(messenger_ready_slo_total[2m])) >= 5)", + "for": "30s", + "labels": { + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "summary": "ready SLO budget burning above 3x", + "runbook": "infra/otel/RUNBOOK.md" + } + }, + { + "alert": "QueueLag", + "expr": "max(messenger_queue_oldest_age_seconds) > 30", + "for": "20s", + "labels": { + "severity": "warning" + }, + "annotations": { + "summary": "Unfinished attachment age exceeds 30s", + "runbook": "infra/otel/RUNBOOK.md" + } + }, + { + "alert": "ObservationsStale", + "expr": "time() - max(messenger_observation_timestamp) > 20 or absent(messenger_observation_timestamp)", + "for": "20s", + "labels": { + "severity": "warning" + }, + "annotations": { + "summary": "No fresh dispatcher observations", + "runbook": "infra/otel/RUNBOOK.md" + } + }, + { + "record": "messenger:api_error_ratio:1h", + "expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[1h])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[1h]))" + }, + { + "record": "messenger:ready_error_ratio:1h", + "expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[1h])) / sum(rate(messenger_ready_slo_total[1h]))" + } + ] + } + ] +} diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..b1f93fb --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,65 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 180 + spike_limit_mib: 40 + batch: + timeout: 1s + send_batch_size: 128 + send_batch_max_size: 256 +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + otlphttp/tempo: + endpoint: http://tempo:4318 + timeout: 2s + retry_on_failure: + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + sending_queue: + queue_size: 32 + num_consumers: 2 + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 2s + retry_on_failure: + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + sending_queue: + queue_size: 32 + num_consumers: 2 +extensions: + health_check: + endpoint: 0.0.0.0:13133 +service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + extensions: [health_check] + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] diff --git a/infra/otel/dashboards/messenger.json b/infra/otel/dashboards/messenger.json new file mode 100644 index 0000000..0aea20d --- /dev/null +++ b/infra/otel/dashboards/messenger.json @@ -0,0 +1,770 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger \u2014 lab6", + "schemaVersion": 39, + "version": 2, + "refresh": "5s", + "time": { + "from": "now-30m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "HTTP RPS", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (route) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 2, + "title": "HTTP 5xx ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger:api_error_ratio:1m", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 3, + "title": "HTTP p50/p95/p99", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "0.5", + "expr": "histogram_quantile(0.5,sum by (le)(rate(messenger_http_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))", + "legendFormat": "0.5" + }, + { + "refId": "0.95", + "expr": "histogram_quantile(0.95,sum by (le)(rate(messenger_http_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))", + "legendFormat": "0.95" + }, + { + "refId": "0.99", + "expr": "histogram_quantile(0.99,sum by (le)(rate(messenger_http_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))", + "legendFormat": "0.99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "description": "Business routes only; histogram quantiles across all API replicas. Bucket interpolation is approximate." + }, + { + "id": 4, + "title": "Job outcomes", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 5, + "title": "Processing p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_duration_seconds_bucket[1m])))", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 6, + "title": "Time to ready p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_ready_duration_seconds_bucket[1m])))", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 7, + "title": "RabbitMQ ready messages", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_queue_depth", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 8, + "title": "Logical pending jobs / outbox", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_jobs_pending", + "legendFormat": "jobs" + }, + { + "refId": "B", + "expr": "messenger_outbox_pending", + "legendFormat": "outbox" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 9, + "title": "Oldest unfinished job", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_queue_oldest_age_seconds", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 10, + "title": "DB connections used/capacity/waiting", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name)(messenger_db_pool_used)", + "legendFormat": "used {{service_name}}" + }, + { + "refId": "B", + "expr": "sum by (service_name)(messenger_db_pool_capacity)", + "legendFormat": "capacity {{service_name}}" + }, + { + "refId": "C", + "expr": "sum by (service_name)(messenger_db_pool_waiting)", + "legendFormat": "waiting {{service_name}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 11, + "title": "Worker busy/capacity", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_worker_busy{service_name=\"messenger-worker\"}", + "legendFormat": "busy" + }, + { + "refId": "B", + "expr": "messenger_worker_capacity{service_name=\"messenger-worker\"}", + "legendFormat": "capacity" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 12, + "title": "Process memory RSS", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_process_memory", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 13, + "title": "CPU by service (% of one core)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "100 * sum by(service_name)(rate(messenger_process_cpu[1m]))", + "legendFormat": "{{service_name}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 14, + "title": "API burn rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger:api_error_ratio:1m / 0.01", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 15, + "title": "Ready burn rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 56, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger:ready_error_ratio:1m / 0.01", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 16, + "title": "Alerts", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 56, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "ALERTS{alertstate=\"firing\"}", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 17, + "title": "Collector export failures", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 64, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "rate(otelcol_exporter_send_failed_spans_total[1m])", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 18, + "title": "Observation freshness", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 64, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "time() - messenger_observation_timestamp", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 19, + "title": "API SLI \u2014 1 hour", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 72, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "1 - messenger:api_error_ratio:1h" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 20, + "title": "Ready SLI \u2014 1 hour", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 72, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "1 - messenger:ready_error_ratio:1h" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 21, + "title": "Exporter bounded queue", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 80, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "otelcol_exporter_queue_size", + "legendFormat": "size {{exporter}}" + }, + { + "refId": "B", + "expr": "otelcol_exporter_queue_capacity", + "legendFormat": "capacity {{exporter}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 22, + "title": "DB lock waiters", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 80, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_db_locks_waiting" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } + ] +} diff --git a/infra/otel/grafana/dashboards/default.yaml b/infra/otel/grafana/dashboards/default.yaml new file mode 100644 index 0000000..0f96809 --- /dev/null +++ b/infra/otel/grafana/dashboards/default.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 +providers: + - name: Messenger + folder: Lab6 + type: file + options: + path: /var/lib/grafana/dashboards diff --git a/infra/otel/grafana/datasources/default.yaml b/infra/otel/grafana/datasources/default.yaml new file mode 100644 index 0000000..3ab104b --- /dev/null +++ b/infra/otel/grafana/datasources/default.yaml @@ -0,0 +1,35 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 5s + - name: Loki + uid: loki + type: loki + url: http://loki:3100 + jsonData: + derivedFields: + - name: TraceID + matcherRegex: trace_id + matcherType: label + datasourceUid: tempo + url: '$${__value.raw}' + maxLines: 1000 + - name: Tempo + uid: tempo + type: tempo + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + tags: [{key: 'service.name', value: 'service_name'}] + spanStartTimeShift: '-1m' + spanEndTimeShift: '1m' + filterByTraceID: true + filterBySpanID: false + nodeGraph: + enabled: true diff --git a/infra/otel/loki.yaml b/infra/otel/loki.yaml new file mode 100644 index 0000000..33a2d4d --- /dev/null +++ b/infra/otel/loki.yaml @@ -0,0 +1,37 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 168h + otlp_config: + resource_attributes: + ignore_defaults: true + attributes_config: + - action: index_label + attributes: [service.name, deployment.environment.name] + - action: structured_metadata + attributes: [service.instance.id, service.version] +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem diff --git a/infra/otel/prometheus.yaml b/infra/otel/prometheus.yaml new file mode 100644 index 0000000..f7c8112 --- /dev/null +++ b/infra/otel/prometheus.yaml @@ -0,0 +1,15 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: + - /etc/prometheus/alerts.yaml +scrape_configs: + - job_name: messenger + static_configs: + - targets: ['otel-collector:8889'] + - job_name: collector + static_configs: + - targets: ['otel-collector:8888'] + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] diff --git a/infra/otel/tempo.yaml b/infra/otel/tempo.yaml new file mode 100644 index 0000000..71eda97 --- /dev/null +++ b/infra/otel/tempo.yaml @@ -0,0 +1,22 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +ingester: + max_block_duration: 5m +compactor: + compaction: + block_retention: 168h +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks +usage_report: + reporting_enabled: false diff --git a/scripts/analyze-backend.py b/scripts/analyze-backend.py new file mode 100644 index 0000000..78106c2 --- /dev/null +++ b/scripts/analyze-backend.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Save bounded-buffer and loss observations for the actual Tempo outage.""" +import datetime as dt,json,re,subprocess,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts')) +from observations import get +x=json.loads((ROOT/'evidence/slo-experiment.json').read_text()) +if not x.get('completed'):raise SystemExit('Experiment has not completed successfully') +p=next(p for p in x['phases'] if p['name']=='tempo_unavailable');finish=next(p['end'] for p in x['phases'] if p['name']=='final_drain') +metrics=['otelcol_receiver_accepted_spans_total','otelcol_exporter_sent_spans_total','otelcol_exporter_send_failed_spans_total','otelcol_exporter_enqueue_failed_spans_total','otelcol_receiver_refused_spans_total','otelcol_exporter_queue_size','otelcol_exporter_queue_capacity'] +def at(name,t): + rows=get('prometheus','/api/v1/query',{'query':name,'time':str(t)})['data']['result'] + return [r for r in rows if name.startswith('otelcol_receiver') or r['metric'].get('exporter')=='otlphttp/tempo'] +before={n:at(n,p['start']) for n in metrics};after={n:at(n,finish) for n in metrics} +def total(rows):return sum(float(r['value'][1]) for r in rows) +deltas={n:total(after[n])-total(before[n]) for n in metrics if n.endswith('_total')} +samples=get('prometheus','/api/v1/query_range',{'query':'otelcol_exporter_queue_size{exporter="otlphttp/tempo"}','start':str(p['start']),'end':str(finish),'step':'5'})['data']['result'] +peak=max((float(v[1]) for r in samples for v in r['values']),default=0) +raw=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'logs','--no-log-prefix','--since',dt.datetime.fromtimestamp(p['start'],dt.timezone.utc).isoformat(),'otel-collector'],text=True) +drops=[] +for line in raw.splitlines(): + try:d=json.loads(line[line.index('{'):]) + except (ValueError,KeyError):continue + if d.get('otelcol.component.id')=='otlphttp/tempo' and ('dropped_items' in d or 'rejected_items' in d): + drops.append({k:d[k] for k in ['dropped_items','rejected_items','otelcol.component.id','otelcol.signal'] if k in d}) +mem=[] +for record in x['resources']: + if p['start']<=record['at']<=finish: + for c in record['containers']: + if 'otel-collector' in c['Name']: + m=re.match(r'([0-9.]+)([A-Za-z]+)',c['MemUsage']);scale={'B':1,'KiB':1024,'MiB':1024**2,'GiB':1024**3}.get(m[2],1);mem.append(float(m[1])*scale) +out={'outage_seconds':p['end']-p['start'],'before':before,'after_drain':after,'counter_deltas':deltas,'collector_accepted_minus_sent_estimate':deltas['otelcol_receiver_accepted_spans_total']-deltas['otelcol_exporter_sent_spans_total'],'estimate_note':'Background health/observer spans remain active; balance is approximate by a few in-flight batches. SDK-side losses are not included; send_failed counts attempts, not unique spans.','peak_exporter_queue':peak,'queue_samples':samples,'collector_memory_peak_bytes':max(mem,default=0),'collector_memory_limit_bytes':256*1024**2,'drop_records':drops,'drop_records_items':sum(d.get('dropped_items',0) for d in drops),'message_persistence':x.get('message_persistence'),'attachment_states':x.get('attachment_states')} +(ROOT/'evidence/backend-outage.json').write_text(json.dumps(out,indent=2)+'\n') +print('Saved backend outage balance, queue, memory and business persistence evidence') diff --git a/scripts/bootstrap-broker.py b/scripts/bootstrap-broker.py new file mode 100644 index 0000000..aa412e1 --- /dev/null +++ b/scripts/bootstrap-broker.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""Create synthetic local broker credentials, preserving existing settings.""" +from pathlib import Path +import secrets +from urllib.parse import quote +root=Path(__file__).resolve().parents[1] +env=root/'.env.lab6' +values=dict(line.split('=',1) for line in env.read_text().splitlines() if '=' in line and not line.startswith('#')) +password=values.get('BROKER_PASSWORD') +if not password: + password=secrets.token_hex(24) + with env.open('a') as f: f.write('\nBROKER_PASSWORD='+password+'\n') +p=root/'secrets/lab6/broker_url' +if p.exists(): p.chmod(0o600) +p.write_text('amqp://messenger:'+quote(password,safe='')+'@broker:5672/\n') +p.chmod(0o444) +print('Local broker credentials ready.') diff --git a/scripts/bootstrap-storage.py b/scripts/bootstrap-storage.py new file mode 100644 index 0000000..dcb458b --- /dev/null +++ b/scripts/bootstrap-storage.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Generate local synthetic storage credentials without rotating existing keys.""" +import json +import os +from pathlib import Path +import secrets + +root=Path(__file__).resolve().parents[1]/'secrets'/'lab6' +root.mkdir(parents=True,exist_ok=True) +for name in ('s3_admin_access','s3_admin_secret','s3_api_access','s3_api_secret'): + p=root/name + if not p.exists(): + p.write_text(secrets.token_hex(24)+'\n') + p.chmod(0o444) # Non-root containers must read these local synthetic secrets. +bucket=os.environ.get('S3_BUCKET','messenger-attachments') +identities=[] +for role,actions in [('admin',['Admin','Read','Write','List','Tagging']),('api',['Read:'+bucket,'Write:'+bucket,'List:'+bucket])]: + identities.append({'name':role,'credentials':[{'accessKey':(root/('s3_'+role+'_access')).read_text().strip(),'secretKey':(root/('s3_'+role+'_secret')).read_text().strip()}],'actions':actions}) +p=root/'s3.json' +content=json.dumps({'identities':identities})+'\n' +if not p.exists() or p.read_text()!=content: + if p.exists(): p.chmod(0o600) + p.write_text(content) +p.chmod(0o444) +print('Local storage credentials ready; existing keys preserved.') diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 0000000..b989718 --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,30 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets/lab6 +if [ ! -f .env.lab6 ]; then + { printf 'POSTGRES_PASSWORD='; openssl rand -hex 24 + printf 'REDIS_PASSWORD='; openssl rand -hex 24 + printf 'HOST_PORT=18086\nAPI_REPLICAS=2\nSESSION_TTL_SECONDS=14400\n' + } > .env.lab6 +fi +if [ ! -f secrets/lab6/private_key.pem ]; then + openssl genrsa -out secrets/lab6/private_key.pem 2048 2>/dev/null +fi +if [ ! -f secrets/lab6/public_key.pem ]; then + openssl rsa -in secrets/lab6/private_key.pem -pubout -out secrets/lab6/public_key.pem 2>/dev/null +fi +# Only synthetic stand keys, readable by the non-root application inside containers. +chmod 755 secrets secrets/lab6 +chmod 444 secrets/lab6/*.pem +printf '%s\n' 'Local lab6 settings and synthetic keys ready; existing files preserved.' + +python3 scripts/bootstrap-storage.py + +python3 scripts/bootstrap-broker.py + +if [ ! -f secrets/lab6/grafana_password ]; then + openssl rand -hex 24 > secrets/lab6/grafana_password +fi +chmod 444 secrets/lab6/grafana_password diff --git a/scripts/capture-environment.py b/scripts/capture-environment.py new file mode 100644 index 0000000..31c0b58 --- /dev/null +++ b/scripts/capture-environment.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Record reproducibility metadata, with an allowlist excluding all secrets.""" +import hashlib,json,platform,subprocess,time +from pathlib import Path +root=Path(__file__).resolve().parents[1] +def cmd(*args):return subprocess.check_output(args,cwd=root,text=True).strip() +paths=[p for d in ['service','infra','scripts','tests','contracts','student-tests'] for p in (root/d).rglob('*') if p.is_file() and '__pycache__' not in p.parts] +paths += [root/x for x in ['Dockerfile','compose.yaml','compose.faults.yaml','Makefile']] +h=hashlib.sha256() +for p in sorted(paths):h.update(str(p.relative_to(root)).encode());h.update(p.read_bytes()) +containers=[json.loads(x) for x in cmd('docker','compose','--env-file','.env.lab6','ps','--format','json').splitlines()] +ids=[c['ID'] for c in containers] +inspect=json.loads(cmd('docker','inspect',*ids)) +rows=[{'service':c['Config']['Labels']['com.docker.compose.service'],'image':c['Config']['Image'],'image_id':c['Image'],'started_at':c['State']['StartedAt'],'memory_limit':c['HostConfig']['Memory'],'nano_cpus':c['HostConfig']['NanoCpus'],'mounts':[{'type':m['Type'],'name':m.get('Name'),'destination':m['Destination']} for m in c['Mounts'] if m['Type']=='volume']} for c in inspect] +info=json.loads(cmd('docker','info','--format','{{json .}}')) +out={'timestamp':time.time(),'revision':cmd('git','rev-parse','HEAD'),'working_tree_dirty':bool(cmd('git','status','--porcelain')),'source_sha256':h.hexdigest(),'platform':platform.platform(),'go':cmd('go','version'),'docker':{'server_version':info['ServerVersion'],'cpus':info['NCPU'],'memory':info['MemTotal'],'architecture':info['Architecture']},'containers':rows} +if platform.system()=='Darwin':out['host']={'cpu':cmd('sysctl','-n','machdep.cpu.brand_string'),'memory':cmd('sysctl','-n','hw.memsize')} +(root/'evidence/environment.json').write_text(json.dumps(out,indent=2)+'\n') +print('Saved sanitized environment and source fingerprint') diff --git a/scripts/capture-incidents.py b/scripts/capture-incidents.py new file mode 100644 index 0000000..b3158c9 --- /dev/null +++ b/scripts/capture-incidents.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Passively save stored log/trace evidence during the running SLO experiment.""" +import json,sys,time +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts')) +from observations import get +source=ROOT/'evidence/slo-experiment.json';target=ROOT/'evidence/incident-signals.json' +result={};errors=[] +def logs(query,start,end): + return get('loki','/loki/api/v1/query_range',{'query':query,'start':str(int(start*1e9)),'end':str(int(end*1e9)),'limit':100})['data']['result'] +def summary(d): + rows=[] + for b in d.get('batches',[]): + resource={a['key']:a['value'].get('stringValue') for a in b.get('resource',{}).get('attributes',[])} + for scope in b.get('scopeSpans',[]): + for s in scope.get('spans',[]): + rows.append({'service':resource.get('service.name'),'name':s['name'],'kind':s.get('kind'),'span_id':s.get('spanId'),'parent_span_id':s.get('parentSpanId'),'duration_ms':(int(s['endTimeUnixNano'])-int(s['startTimeUnixNano']))/1e6,'status':s.get('status')}) + return rows +end=time.time()+8000 +while time.time()60 and phase not in result: + rows=logs('{service_name="messenger-api"} |= "http_error"',event['at'],time.time()-3) + ids=[r['stream'].get('trace_id') for r in rows if r['stream'].get('trace_id')] + if ids: + traceid=ids[-1];spans=summary(get('tempo','/api/traces/'+traceid)) + if any(s['name'].startswith('postgresql ') and s['duration_ms']>1000 for s in spans):result[phase]={'captured':time.time(),'observed':last,'trace_id':traceid,'spans':spans,'error_log_events':sum(len(r['values']) for r in rows),'diagnosis':'DB wait dominates server span; HTTP 5xx, not image processing'} + if phase.startswith('worker_long_') and time.time()-event['at']>70 and phase not in result: + worker=logs('{service_name="messenger-worker"} |= "job_received"',event['at'],time.time()-3) + uploads=logs('{service_name="messenger-api"} |= "http_request" | route="/api/v1/chats/{chat_id}/attachments" | status="202"',event['at'],time.time()-3) + result[phase]={'captured':time.time(),'observed':last,'worker_received_events':sum(len(r['values']) for r in worker),'accepted_upload_events':sum(len(r['values']) for r in uploads),'diagnosis':'Queue age and ready burn increase while API accepts uploads and no worker consumption is observed'} + if phase.startswith('worker_recovery_') and phase not in result and time.time()-event['at']>20: + rows=logs('{service_name="messenger-worker"} |= "job_ready"',event['at'],time.time()-3) + ids=[r['stream'].get('trace_id') for r in rows if r['stream'].get('trace_id')] + if ids: + traceid=ids[0];spans=summary(get('tempo','/api/traces/'+traceid)) + if spans:result[phase]={'captured':time.time(),'trace_id':traceid,'spans':spans,'diagnosis':'Worker resumes consumption and publishes S3/DB results'} + target.write_text(json.dumps({'evidence':result,'collection_errors':errors[-20:]},indent=2)+'\n') + if x.get('completed') is not None:break + except Exception as e:errors.append({'at':time.time(),'type':type(e).__name__}) + time.sleep(5) +print('Saved incident observations:',len(result)) diff --git a/scripts/compose.sh b/scripts/compose.sh new file mode 100755 index 0000000..f57fd5d --- /dev/null +++ b/scripts/compose.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec docker compose --env-file .env.lab6 "$@" diff --git a/scripts/experiment-slo.py b/scripts/experiment-slo.py new file mode 100644 index 0000000..c2fd1fe --- /dev/null +++ b/scripts/experiment-slo.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Bounded SLO experiment on the isolated lab6 stand, with raw observations. +No synthetic metrics or time acceleration. Restores every injected fault in finally. +""" +import argparse,concurrent.futures,hashlib,json,math,os,signal,statistics,subprocess,sys,threading,time,urllib.request,urllib.error,uuid +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')] +from client import Client +from observations import get,query,scalar +C=[str(ROOT/'scripts/compose.sh')] +def compose(*a):return subprocess.run(C+list(a),cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=90) +def start_service(name): + ids=subprocess.check_output(C+['ps','-a','-q',name],text=True).split() + if not ids:raise RuntimeError('Missing existing '+name+' container') + subprocess.run(['docker','start',*ids],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=30) +def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip() +def percentile(xs,q):return sorted(xs)[max(0,math.ceil(len(xs)*q)-1)] if xs else None +class Traffic: + def __init__(self): + c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap() + self.chat=chat;self.token=a['token'];self.actor=a;self.client=c + self.fixture=(ROOT/'tests/fixtures/sample.png').read_bytes() + self.lock=threading.Lock();self.rows=[];self.attachments=[];self.messages=[];self.dropped=0;self.stop=threading.Event();self.slots=threading.BoundedSemaphore(64) + def request(self,index,scheduled,upload=False): + try: + path='/api/v1/chats/'+self.chat['id'] + headers={'Authorization':'Bearer '+self.token};body=None + if upload:method='POST';path+='/attachments?filename=sample.png';body=self.fixture;headers['Content-Type']='image/png' + elif index%10==0:method='POST';path+='/messages';body=b'{"text":"synthetic-slo"}';headers.update({'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex}) + else:method='GET';path+='/messages?limit=50' + r=urllib.request.Request('http://127.0.0.1:18086'+path,data=body,headers=headers,method=method) + status=0;att=None;message_id=None + try: + try:response=urllib.request.urlopen(r,timeout=6) + except urllib.error.HTTPError as e:response=e + with response: + status=response.status;data=response.read() + if upload and status==202:att=json.loads(data)['id'] + if not upload and method=='POST' and status==201:message_id=json.loads(data)['id'] + except (OSError,TimeoutError):pass + with self.lock: + self.rows.append((time.time(),upload,status,(time.monotonic()-scheduled)*1000)) + if att:self.attachments.append(att) + if message_id:self.messages.append(message_id) + finally:self.slots.release() + def run(self): + start=time.monotonic();i=0 + with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool: + while not self.stop.is_set(): + scheduled=start+i*.1 + if self.stop.wait(max(0,scheduled-time.monotonic())):break + if self.slots.acquire(False):pool.submit(self.request,i,scheduled) + else: + with self.lock:self.dropped+=1 + if i%20==0: + if self.slots.acquire(False):pool.submit(self.request,i,scheduled,True) + else: + with self.lock:self.dropped+=1 + i+=1 + def summary(self): + with self.lock:rows=list(self.rows);dropped=self.dropped + xs=[r[3] for r in rows if 200<=r[2]<300] + return {'requests':len(rows),'success':sum(200<=r[2]<300 for r in rows),'errors':sum(not 200<=r[2]<300 for r in rows),'generator_dropped':dropped,'uploads_accepted':sum(r[1] and r[2]==202 for r in rows),'latency_success_ms':{str(q):percentile(xs,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}} + +def main(): + def interrupted(*_):raise KeyboardInterrupt() + signal.signal(signal.SIGTERM,interrupted) + p=argparse.ArgumentParser();p.add_argument('--warmup',type=int,default=900);p.add_argument('--long',type=int,default=240);p.add_argument('--cooldown',type=int,default=660);p.add_argument('--repeats',type=int,default=3);p.add_argument('--backend',type=int,default=300);p.add_argument('--output',default='evidence/slo-experiment.json');a=p.parse_args() + out={'started':time.time(),'revision':subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),'parameters':vars(a),'profile':{'message_rps':10,'read_fraction':.9,'image_interval_seconds':2,'max_inflight':64,'transparent_retries':False},'events':[],'observations':[],'resources':[],'phases':[]} + out['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab6','--format','{{.Id}}'],text=True).strip() + target=ROOT/a.output + def save():target.write_text(json.dumps(out,indent=2)+'\n') + traffic=Traffic();t=threading.Thread(target=traffic.run);t.start() + def phase(name,seconds): + at=time.time();out['events'].append({'phase':name,'at':at});print(name,seconds,'seconds',flush=True);save() + before=traffic.summary();nextstats=0;nextprint=0 + while time.time()=nextstats: + ids=subprocess.check_output(C+['ps','-q','api','worker','dispatcher','otel-collector'],text=True).split() + if ids: + raw=subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',*ids],text=True,timeout=20) + out['resources'].append({'at':time.time(),'phase':name,'containers':[json.loads(x) for x in raw.splitlines()]}) + nextstats=now+30 + if now>=nextprint:print(name,round(time.time()-at),'s',traffic.summary()['requests'],'requests',flush=True);nextprint=now+60 + save();time.sleep(min(5,max(0,at+seconds-time.time()))) + with traffic.lock: rows=[r for r in traffic.rows if at<=r[0]<=time.time()] + lat=[r[3] for r in rows if 200<=r[2]<300] + elapsed=time.time()-at;success=sum(200<=r[2]<300 for r in rows) + out['phases'].append({'name':name,'start':at,'end':time.time(),'before':before,'after':traffic.summary(),'successful_rps':success/elapsed,'error_ratio':1-success/len(rows) if rows else None,'latency_success_ms':{str(q):percentile(lat,q) for q in [.5,.95,.99]},'statuses':{str(s):sum(r[2]==s for r in rows) for s in sorted(set(r[2] for r in rows))}});save() + def worker(duration,name): + compose('stop','worker') + try:phase(name,duration) + finally:start_service('worker') + def database(duration,name): + # A separate bounded transaction blocks only the local message table. + proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1','-c',f"SET application_name='lab6_slo_injection'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep({duration}); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + try: + deadline=time.time()+10 + while time.time()=2) + marker.unlink() + ready=c.wait_ready(a,att) + def complete(): + spans=trace_summary(get('tempo','/api/traces/'+traceid)) + return spans if any(s['service']=='messenger-api' and s['name'].startswith('POST ') for s in spans) and any(s['name']=='attachment.process' for s in spans) and any(s['name']=='s3.put' and s['service']=='messenger-worker' for s in spans) else None + spans=wait(complete) + def logs(): + d=get('loki','/loki/api/v1/query_range',{'query':'{service_name=~"messenger-.+"} | trace_id="'+traceid+'"','start':str(int((time.time()-300)*1e9)),'end':str(int(time.time()*1e9)),'limit':'1000'}) + rows=d['data']['result'] + return rows if {'messenger-api','messenger-worker'}.issubset({r['stream'].get('service_name') for r in rows}) else None + rows=wait(logs) + events=[] + for r in rows: + stream=r['stream'];events.extend({'service':stream.get('service_name'),'trace_id':stream.get('trace_id'),'span_id':stream.get('span_id'),'event':line,'timestamp':t} for t,line,*_ in r['values']) + check(any(s['kind'] in ['SPAN_KIND_PRODUCER',4] for s in spans),'producer span missing') + check(any(s['kind'] in ['SPAN_KIND_CONSUMER',5] for s in spans),'consumer span missing') + producers={s['spanId'] for s in spans if s['kind'] in ['SPAN_KIND_PRODUCER',4]} + consumers=[s for s in spans if s['kind'] in ['SPAN_KIND_CONSUMER',5]] + check(all(s['parentSpanId'] in producers for s in consumers),'consumer parent is not a producer') + check(sql("SELECT split_part(traceparent,'-',2) FROM attachment_jobs WHERE id='%s'"%jid)==traceid,'durable context was lost') + results.append({'trace_id' :traceid,'retry':retry,'attachment_id':att['id'],'job_id':jid,'attempt':int(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%jid)),'spans':spans,'logs':events}) + print('PASS: correlated upload, retry='+str(retry),flush=True) + finally: + if marker:marker.unlink(missing_ok=True) + if retry:compose('up','-d','--no-deps','worker') +(ROOT/'evidence/correlation.json').write_text(json.dumps(results,indent=2)+'\n') diff --git a/scripts/prove-first-spike.py b/scripts/prove-first-spike.py new file mode 100644 index 0000000..112f54e --- /dev/null +++ b/scripts/prove-first-spike.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Confirm that Prometheus sees the first 5xx spike after zero-series initialization.""" +import concurrent.futures,json,subprocess,sys,time +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')] +from client import Client,check +from observations import query,scalar +C=[str(ROOT/'scripts/compose.sh')] +def sql(q):return subprocess.check_output(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',q],text=True,timeout=10).strip() +c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap() +before=query('messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}') +proc=subprocess.Popen(C+['exec','-T','postgres','psql','-U','messenger','-d','messenger','-c',"SET application_name='lab6_counter_probe'; BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(5); ROLLBACK;"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) +def request():return c.call('GET','/api/v1/chats/'+chat['id']+'/messages',expected=[200,500,503],actor=a)['status'] +try: + end=time.time()+5 + while time.time()0,'first spike missing from rate/increase') +out={'at':time.time(),'before':before,'after':query('messenger_http_requests_total{route="/api/v1/chats/{chat_id}/messages",status_class="5xx"}'),'increase_1m':increase,'statuses':{str(s):statuses.count(s) for s in set(statuses)},'passed':True} +(ROOT/'evidence/first-spike.json').write_text(json.dumps(out,indent=2)+'\n');print('PASS: first 5xx spike appears in Prometheus increase:',round(increase,2)) diff --git a/scripts/prove-persistence.py b/scripts/prove-persistence.py new file mode 100644 index 0000000..6869aa8 --- /dev/null +++ b/scripts/prove-persistence.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Attach container/volume evidence to the existing public persistence scenario.""" +import json,subprocess,time +from pathlib import Path +root=Path(__file__).resolve().parents[1] +compose=[str(root/'scripts/compose.sh')] +def snapshot(): + ids=subprocess.check_output(compose+['ps','-q'],text=True).split() + rows=json.loads(subprocess.check_output(['docker','inspect',*ids],text=True)) + return {r['Name'].lstrip('/'):{'service':r['Config']['Labels']['com.docker.compose.service'],'started_at':r['State']['StartedAt'],'volumes':sorted(m['Name'] for m in r['Mounts'] if m['Type']=='volume')} for r in rows} +before=snapshot();start=time.time() +p=subprocess.run(['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh'],cwd=root,text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) +(root/'evidence/persistence.txt').write_text(p.stdout) +after=snapshot();stores=['postgres','redis','s3','broker','prometheus','loki','tempo','grafana'] +checks={} +for service in stores: + old=next(v for v in before.values() if v['service']==service);new=next(v for v in after.values() if v['service']==service) + checks[service]={'restarted':old['started_at']!=new['started_at'],'same_volumes':old['volumes']==new['volumes']} +out={'started':start,'finished':time.time(),'command':['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh'],'returncode':p.returncode,'before':before,'after':after,'checks':checks} +(root/'evidence/persistence-processes.json').write_text(json.dumps(out,indent=2)+'\n') +if p.returncode or not all(v['restarted'] and v['same_volumes'] for v in checks.values()):raise SystemExit('FAIL: persistence or actual store restart evidence') +print('PASS: persistence, real restart of all stores, unchanged volume identities') diff --git a/scripts/prove-signals.py b/scripts/prove-signals.py new file mode 100644 index 0000000..bd512f0 --- /dev/null +++ b/scripts/prove-signals.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Inspect real stored signals, label cardinality and synthetic privacy canaries.""" +import json,sys,time,uuid +from pathlib import Path +root=Path(__file__).resolve().parents[1];sys.path[:0]=[str(root/'tests'),str(root/'scripts')] +from client import Client,check +from observations import get,grafana,query,collect +c=Client('http://127.0.0.1:18086');a,b,e,chat=c.bootstrap() +marker='private_canary_'+uuid.uuid4().hex +c.message(a,chat,marker) +c.call('POST','/api/v1/chats',400,actor=a,body={marker:marker}) +att,_=c.attachment(a,chat);c.wait_ready(a,att) +time.sleep(12) +series=query('{__name__=~"messenger_.*"}') +for row in series: + labels=row['metric'] + check(not set(labels).intersection({'user_id','chat_id','message_id','attachment_id','request_id','filename','url'}),'high-cardinality metric label') + check(not __import__('re').search(r'[0-9a-f]{8}-[0-9a-f]{4}-',labels.get('route','')),'raw route contains UUID') +required=['messenger_http_requests_total','messenger_http_duration_seconds_bucket','messenger_jobs_total','messenger_job_duration_seconds_bucket','messenger_job_ready_duration_seconds_bucket','messenger_db_pool_used','messenger_db_pool_waiting','messenger_worker_busy','messenger_worker_capacity','messenger_process_memory','messenger_process_cpu','messenger_queue_depth','messenger_queue_oldest_age_seconds','messenger_ready_slo_total'] +found={r['metric']['__name__'] for r in series} +check(set(required)<=found,'missing required metrics: '+str(set(required)-found)) +logs=get('loki','/loki/api/v1/query_range',{'query':'{service_name=~"messenger-.+"} |= "'+marker+'"','start':str(int((time.time()-180)*1e9)),'end':str(int(time.time()*1e9))}) +check(not logs['data']['result'],'canary leaked into logs') +# Search trace attributes; HTTP bodies and headers are never recorded. +traces=get('tempo','/api/search',{'q':'{ resource.service.name = "messenger-api" }','limit':10}) +trace_rows=[] +for tr in traces.get('traces',[]): + d=get('tempo','/api/traces/'+tr['traceID']);check(marker not in json.dumps(d),'canary leaked into traces');trace_rows.append(tr['traceID']) +check(bool(trace_rows),'trace search returned no observations') +# Verify that the imported dashboard references valid expressions. +dashboard=grafana('/api/dashboards/uid/messenger-lab6')['dashboard'] +check(len(dashboard['panels'])>=18,'dashboard not provisioned') +provisioned=grafana('/api/datasources');check({'prometheus','loki','tempo'}<=set(d['uid'] for d in provisioned),'datasources not provisioned') +for panel in dashboard['panels']: + for target in panel.get('targets',[]):query(target['expr']) +out=collect();out.update({'checks':{'required_metrics_present':True,'forbidden_labels_absent':True,'raw_uuid_routes_absent':True,'synthetic_canary_absent_in_logs':True,'synthetic_canary_absent_in_traces':True,'dashboard_expressions_valid':True},'series_count':len(series),'trace_ids_checked':trace_rows}) +(root/'evidence/signals.json').write_text(json.dumps(out,indent=2)+'\n');print('PASS: stored metrics, safe labels, privacy canaries, trace search and dashboard queries') diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..89f53e3 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +./scripts/compose.sh stop api lb worker dispatcher +./scripts/compose.sh restart postgres redis s3 broker prometheus loki tempo otel-collector grafana +./scripts/compose.sh run --rm -T s3-init +./scripts/compose.sh up -d --wait --wait-timeout 90 api lb worker dispatcher diff --git a/scripts/rolling-update.py b/scripts/rolling-update.py new file mode 100644 index 0000000..1db08f1 --- /dev/null +++ b/scripts/rolling-update.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Drain and replace API processes individually, retaining other container IDs.""" +import json,os,subprocess,time +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +def run(args,input=None): + p=subprocess.run(args,cwd=ROOT,input=input,text=True,capture_output=True,timeout=100) + if p.returncode:raise RuntimeError(p.stderr.strip()[:300]) + return p.stdout.strip() +def runtime(lb,cmd):return run(['docker','exec','-i',lb,'nc','-w','1','127.0.0.1','9999'],cmd+'\n') +def ids():return run(['./scripts/compose.sh','ps','-q','api']).split() +def states(lb): + lines=runtime(lb,'show servers state').splitlines();head=lines[1].lstrip('# ').split() + return [dict(zip(head,l.split())) for l in lines[2:] if l.strip()] +def main(): + initial=ids() + if len(initial)<2:raise SystemExit('rolling update requires at least two running API replicas') + lb=run(['./scripts/compose.sh','ps','-q','lb']) + for old in initial: + info=json.loads(run(['docker','inspect',old]))[0] + ip=next(iter(info['NetworkSettings']['Networks'].values()))['IPAddress'] + backend=next(s['srv_name'] for s in states(lb) if s['srv_addr']==ip and s['be_name']=='api') + runtime(lb,f'set server api/{backend} state drain') + try: + deadline=time.monotonic()+10 + while True: + # Existing transactions get a chance to finish before SIGTERM. + import csv,io + raw=run(['docker','exec',lb,'wget','-qO-','http://127.0.0.1:8404/stats;csv']) + row=next(r for r in csv.DictReader(io.StringIO(raw)) if r['# pxname']=='api' and r['svname']==backend) + if int(row['scur'])==0:break + if time.monotonic()>deadline:raise RuntimeError('backend did not drain') + time.sleep(.1) + before=set(ids());run(['docker','stop','--time','8',old]);run(['docker','rm',old]) + run(['./scripts/compose.sh','up','-d','--no-deps','--no-recreate','--scale',f'api={len(initial)}','api']) + after=set(ids()) + if not (before-{old}).issubset(after):raise RuntimeError('another replica was replaced unexpectedly') + new=(after-before).pop();deadline=time.monotonic()+60 + while run(['docker','inspect','--format','{{.State.Health.Status}}',new])!='healthy': + if time.monotonic()>deadline:raise RuntimeError('replacement failed readiness') + time.sleep(.5) + print(json.dumps({'old_container':old,'new_container':new,'name':info['Name'],'drained_backend':backend}),flush=True) + finally:runtime(lb,f'set server api/{backend} state ready') + time.sleep(3) # allow DNS refresh and a successful active readiness check +if __name__=='__main__':main() diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..c56d628 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/rolling-update.py diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..8d21383 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?missing target alias}" +for id in $(./scripts/compose.sh ps -q api); do + current=$(docker exec "$id" wget -T 2 -qO- http://127.0.0.1:8080/internal/metrics | python3 -c 'import json,sys; print(json.load(sys.stdin)["instance_id"])') + if [ "$current" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 8 "$id" >/dev/null + python3 - "$id" "$TARGET_INSTANCE_ID" <<'PYLOG' +import json,sys,time +from pathlib import Path +with Path('evidence/failover-action.jsonl').open('a') as f: + f.write(json.dumps({'at':time.time(),'container':sys.argv[1],'instance_id':sys.argv[2],'action':'docker stop','project':'mtusi2026lab6'})+'\n') +PYLOG + exit 0 + fi +done +printf '%s\n' 'Target does not match a running lab6 API process' >&2 +exit 1 diff --git a/scripts/summarize-slo.py b/scripts/summarize-slo.py new file mode 100644 index 0000000..153c908 --- /dev/null +++ b/scripts/summarize-slo.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Derive incident timing from recorded observations, without inventing missing data.""" +import json +from pathlib import Path +root=Path(__file__).resolve().parents[1] +x=json.loads((root/'evidence/slo-experiment.json').read_text()) +if not x.get('completed'):raise SystemExit('Experiment has not completed successfully') +rows=x['observations'];events=x['events'];results=[] +for i,event in enumerate(events): + name=event['phase'] + if not any(name.startswith(s) for s in ['worker_short_','db_short_','worker_long_','db_long_']):continue + if 'recovery' in name:continue + prefix=name.rsplit('_',1)[0];start=event['at'] + phase=next(p for p in x['phases'] if p['name']==name);recovered=phase['end'] + nextfault=next((e['at'] for e in events[i+1:] if ('_short_' in e['phase'] or '_long_' in e['phase']) and 'recovery' not in e['phase']),x.get('finished',float('inf'))) + observed=[r for r in rows if start<=r['at']max(first,recovered) and not firing(r)),None) + alerts[alert]={'detection_seconds':None if first is None else first-start,'resolved_after_restore_seconds':None if resolved is None else resolved-recovered,'fired':first is not None} + results.append({'phase':name,'duration':recovered-start,'alerts':alerts,'successful_rps':phase['successful_rps'],'error_ratio':phase['error_ratio'],'latency_success_ms':phase['latency_success_ms']}) +summary={'completed':x.get('completed',False),'duration_seconds':x.get('finished',0)-x['started'],'parameters':x['parameters'],'incidents':results,'traffic':x.get('traffic'),'attachment_states':x.get('attachment_states'),'image_id':x.get('image_id')} +(root/'evidence/slo-summary.json').write_text(json.dumps(summary,indent=2)+'\n') +print(json.dumps(summary,indent=2)) diff --git a/scripts/test-storage.sh b/scripts/test-storage.sh new file mode 100755 index 0000000..9cb8307 --- /dev/null +++ b/scripts/test-storage.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +mkdir -p bin +(cd service && GOOS=linux GOARCH=$(docker info --format '{{.Architecture}}' | sed 's/aarch64/arm64/;s/x86_64/amd64/') CGO_ENABLED=0 go test -c -o ../bin/storage.test ./internal/adapter/repository/storage) +database="lab5_atomic_$(date +%s)_$$" +./scripts/compose.sh exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 -c "CREATE DATABASE $database" >/dev/null +trap './scripts/compose.sh exec -T postgres psql -U messenger -d messenger -c "DROP DATABASE IF EXISTS $database WITH (FORCE)" >/dev/null' EXIT +id=$(./scripts/compose.sh ps -q api | head -n 1) +docker cp bin/storage.test "$id:/tmp/storage.test" >/dev/null +docker exec -e "TEST_DATABASE_NAME=$database" "$id" /tmp/storage.test -test.v diff --git a/scripts/wait-ready.py b/scripts/wait-ready.py new file mode 100644 index 0000000..64be15b --- /dev/null +++ b/scripts/wait-ready.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Wait for the business endpoint and each telemetry backend, not only running processes.""" +import subprocess,time,urllib.request +from pathlib import Path +root=Path(__file__).resolve().parents[1] +checks=['http://otel-collector:13133/','http://prometheus:9090/-/ready','http://loki:3100/ready','http://tempo:3200/ready','http://grafana:3000/api/health'] +end=time.monotonic()+120 +while time.monotonic() 1 && os.Args[1] == "migrate" { + cfg.Postgres.RunMigrations = true + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + return migrator.New(cfg).Start(ctx) + } + name := "messenger-api" + if len(os.Args) > 1 { + name = "messenger-" + os.Args[1] + } + shutdown, err := telemetry.Init(context.Background(), name) + if err != nil { + return err + } + defer shutdown() + attachmentConfig, err := config.LoadAttachment() + if err != nil { + return err + } + if len(os.Args) > 1 && (os.Args[1] == "worker" || os.Args[1] == "dispatcher" || os.Args[1] == "jobs" || os.Args[1] == "outbox") { + return runPipeline(cfg, attachmentConfig, os.Args[1:]) + } + if len(os.Args) > 1 && os.Args[1] == "storage" { + return runStorage(cfg, attachmentConfig, os.Args[2:]) + } + + // API processes never run migrations; the one-shot migrator owns schema changes. + cfg.Postgres.RunMigrations = false + var level slog.Level + if err := level.UnmarshalText([]byte(cfg.Application.LogLevel)); err != nil { + return err + } + // Logging is initialized for every process by telemetry.Init. + instanceID := telemetry.InstanceID + slog.Info("instance_started", "instance_id", instanceID) + pgx := postgres.New(cfg.Postgres) + rx := cache.New(cfg) + postgresMigrator := migrator.New(cfg) + defer pgx.Close() + telemetry.ObservePool(pgx, 0) + defer rx.Close() + tm := txmanager.New(pgx) + provider := jwtprovider.New(cfg) + encryptor := encrypting.New() + + chatsRepo := storage.NewChatsRepo(pgx) + messageRepo := storage.NewMessagesRepo(pgx) + refreshTokensRepo := storage.NewRefreshTokensRepo(pgx) + userRepo := storage.NewUsersRepo(pgx) + + chatCache := outside.NewChatCache(rx) + + authSvc := auth.New(provider, tm, encryptor, userRepo, refreshTokensRepo) + userSvc := user.New(encryptor, userRepo) + chatSvc := chat.New(tm, chatsRepo, messageRepo, chatCache) + objectStore := objectstore.New(attachmentConfig) + attachmentSvc := attachment.New(&storage.AttachmentsRepo{Pool: pgx}, objectStore, attachmentConfig, chatSvc) + pipelineConfig, err := config.LoadPipeline() + if err != nil { + return err + } + outboxSvc := outbox.New(&storage.OutboxRepo{Pool: pgx}, tm, nil) + attachmentSvc.Configure(tm, outboxSvc, pipelineConfig) + + httpRouter := router.New( + middleware.NewAuthMiddleware(authSvc), + middleware.Instance(instanceID), + middleware.RequestIDMiddleware(), + telemetry.HTTP, + middleware.RequestTimeout(cfg.Postgres.QueryTimeout, attachmentConfig.Timeout), + ) + httpRouter.Get("/internal/metrics", func(w stdhttp.ResponseWriter, r *stdhttp.Request) { + stat := pgx.Stat() + _ = json.NewEncoder(w).Encode(map[string]any{"instance_id": instanceID, "acquire_count": stat.AcquireCount(), "acquire_duration_ns": stat.AcquireDuration().Nanoseconds(), "empty_acquire_count": stat.EmptyAcquireCount(), "acquired_conns": stat.AcquiredConns(), "total_conns": stat.TotalConns(), "max_conns": stat.MaxConns()}) + }) + server := http.New(cfg, httpRouter) + requestValidator := validator.NewValidatorImpl() + registerRoutes( + http.NewAuthHandler(httpRouter, authSvc), + http.NewHealthHandler(httpRouter, server, pgx, objectStore), + http.NewChatHandler(httpRouter, provider, requestValidator, chatSvc), + http.NewUserHandler(httpRouter, provider, requestValidator, userSvc), + http.NewAttachmentHandler(httpRouter, provider, attachmentSvc), + ) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + if err = postgresMigrator.Start(ctx); err != nil { + return fmt.Errorf("start postgres migrator: %w", err) + } + if err = server.Start(ctx); err != nil { + return fmt.Errorf("start HTTP server: %w", err) + } + if err = rx.Start(ctx); err != nil { + slog.Warn("redis unavailable at startup; continuing without cache", "error", err) + } + slog.Info("HTTP server started", "address", cfg.Server.BindAddress()) + + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.Server.ShutdownTimeout) + defer cancel() + if err = server.Stop(shutdownCtx); err != nil { + return fmt.Errorf("stop HTTP server: %w", err) + } + if err = postgresMigrator.Close(shutdownCtx); err != nil { + return fmt.Errorf("close postgres migrator: %w", err) + } + if err = rx.Close(); err != nil { + return fmt.Errorf("close redis: %w", err) + } + pgx.Close() + return nil +} + +func registerRoutes(handlers ...http.Handler) { + for _, handler := range handlers { + handler.RegisterRoutes() + } +} diff --git a/service/cmd/app/pipeline.go b/service/cmd/app/pipeline.go new file mode 100644 index 0000000..d712b3f --- /dev/null +++ b/service/cmd/app/pipeline.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "gitea.privcloud.tech/mtusi2026/internal/adapter/broker" + "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage" + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "gitea.privcloud.tech/mtusi2026/internal/service/observation" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "log/slog" + "os" + "os/signal" + "syscall" + "time" +) + +func runPipeline(cfg *config.Config, a *config.Attachment, args []string) error { + c, e := config.LoadPipeline() + if e != nil { + return e + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + pool := postgres.New(cfg.Postgres) + defer pool.Close() + telemetry.ObservePool(pool, c.Concurrency) + tm := txmanager.New(pool) + repo := &storage.AttachmentsRepo{Pool: pool} + rabbit := &broker.Rabbit{URL: c.BrokerURL} + ob := outbox.New(&storage.OutboxRepo{Pool: pool}, tm, rabbit) + svc := attachment.New(repo, objectstore.New(a), a, nil) + svc.Configure(tm, ob, c) + ob.OnDead = svc.PublishFailed + if args[0] == "jobs" || args[0] == "outbox" { + if len(args) < 2 { + return errors.New("operator action required") + } + fs := flag.NewFlagSet(args[1], flag.ContinueOnError) + idText := fs.String("id", "", "job ID (attachment ID for reprocess)") + version := fs.Int("pipeline-version", 2, "new pipeline version") + if e = fs.Parse(args[2:]); e != nil { + return e + } + if args[1] == "list" { + jobs, e := repo.ListJobs(ctx) + if e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(jobs) + } + id, e := uuid.Parse(*idText) + if e != nil { + return e + } + switch args[1] { + case "redeliver": + if os.Getenv("LAB5_FAULTS") != "1" { + return errors.New("redeliver is restricted to the explicit fault profile") + } + var payload []byte + e = tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + payload, e = json.Marshal(j.Message) + return e + }) + if e == nil { + e = rabbit.Publish(ctx, outbox.DeliveryMessage{ID: uuid.New(), AggregateID: id, Payload: payload}) + } + case "replay": + e = svc.Replay(ctx, id) + case "reprocess": + e = svc.Reprocess(ctx, id, *version) + default: + return errors.New("unknown operator action") + } + if e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"accepted": true, "id": id}) + } + if c.BrokerURL == "" { + return errors.New("BROKER_URL_FILE is required") + } + if args[0] == "worker" { + // Recovery continues while the consumer is disconnected or a delivery is paused. + recoveryDone := make(chan struct{}) + defer func() { stop(); <-recoveryDone }() + go func() { + defer close(recoveryDone) + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + if e := svc.Recover(ctx); e != nil { + slog.Warn("job_recovery_failed") + } + if e := svc.Cleanup(ctx); e != nil { + slog.Warn("source_cleanup_failed") + } + } + } + }() + for ctx.Err() == nil { + if e = rabbit.Consume(ctx, c.Concurrency, c.Prefetch, svc.Process); e != nil && ctx.Err() == nil { + slog.Warn("consumer_reconnecting") + } + select { + case <-ctx.Done(): + case <-time.After(time.Second): + } + } + return nil + } + observer := &observation.Service{Repo: &storage.ObservationRepo{Pool: pool}, Publish: func(s observation.Snapshot) { + telemetry.PublishObservation(s.Queued, s.Outbox, s.Good, s.Bad, s.Oldest, s.Locked) + }} + telemetry.RegisterObservation() + nextObservation := time.Time{} + for ctx.Err() == nil { + if time.Now().After(nextObservation) { + if err := observer.Observe(ctx); err != nil { + slog.Warn("observation_unavailable") + } + depth, err := rabbit.Depth(ctx) + if err == nil { + telemetry.QueueDepth.Store(int64(depth)) + } + nextObservation = time.Now().Add(time.Second) + } + _, e = ob.ProcessOnce(ctx) + if e != nil { + slog.Warn("dispatcher_retry") + } + select { + case <-ctx.Done(): + case <-time.After(100 * time.Millisecond): + } + } + return nil +} diff --git a/service/cmd/app/storage.go b/service/cmd/app/storage.go new file mode 100644 index 0000000..09d459c --- /dev/null +++ b/service/cmd/app/storage.go @@ -0,0 +1,183 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "log/slog" + "net/http" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage" + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" +) + +func runStorage(cfg *config.Config, c *config.Attachment, args []string) error { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))) + if len(args) == 0 { + return fmt.Errorf("storage command required") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + store := objectstore.New(c) + if args[0] == "bootstrap" { + var err error + for i := 0; i < 60; i++ { + _, err = store.Client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &c.Bucket}) + var api smithy.APIError + if err == nil || (errors.As(err, &api) && (api.ErrorCode() == "BucketAlreadyOwnedByYou" || api.ErrorCode() == "BucketAlreadyExists")) { + if store.Ping(ctx) == nil { + // HEAD alone can succeed before the restarted master/volume write path. + // This constant key is outside the application's immutable originals prefix. + key := "bootstrap/write-check" + if store.Put(ctx, key, "application/octet-stream", strings.NewReader("ready"), 5) == nil && store.Delete(ctx, key) == nil { + return json.NewEncoder(os.Stdout).Encode(map[string]any{"bucket_ready": true, "write_checked": true}) + } + } + } + time.Sleep(time.Second) + } + return fmt.Errorf("storage bootstrap failed: %w", err) + } + if args[0] == "private-check" { + // Unsigned requests must reach the S3 server and receive an auth failure, + // rather than passing because a network error hid a public object. + fs := flag.NewFlagSet("private-check", flag.ContinueOnError) + key := fs.String("key", "", "known object key") + if e := fs.Parse(args[1:]); e != nil { + return e + } + client := http.Client{Timeout: 5 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + resp, e := client.Get(c.Endpoint + "/" + c.Bucket + "/" + *key) + if e != nil { + return e + } + defer resp.Body.Close() + if resp.StatusCode != 401 && resp.StatusCode != 403 { + return fmt.Errorf("unsigned GET returned %d", resp.StatusCode) + } + _, e = store.Client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: new("forbidden-" + uuid.NewString())}) + var api smithy.APIError + if !errors.As(e, &api) || api.ErrorCode() != "AccessDenied" { + return fmt.Errorf("API key bucket creation was not denied") + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"anonymous_status": resp.StatusCode, "api_create_bucket_denied": true}) + } + if args[0] == "object-check" { + fs := flag.NewFlagSet("object-check", flag.ContinueOnError) + key := fs.String("key", "", "synthetic object key") + if e := fs.Parse(args[1:]); e != nil { + return e + } + head, e := store.Head(ctx, *key) + if objectstore.Missing(e) { + return json.NewEncoder(os.Stdout).Encode(map[string]any{"exists": false}) + } + if e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"exists": true, "last_modified": head.LastModified, "size_bytes": head.ContentLength}) + } + + pool := postgres.New(cfg.Postgres) + defer pool.Close() + svc := attachment.New(&storage.AttachmentsRepo{Pool: pool}, store, c, nil) + jobs, e := config.LoadPipeline() + if e != nil { + return e + } + tm := txmanager.New(pool) + svc.Configure(tm, outbox.New(&storage.OutboxRepo{Pool: pool}, tm, nil), jobs) + switch args[0] { + case "reconcile": + fs := flag.NewFlagSet("reconcile", flag.ContinueOnError) + apply := fs.Bool("apply", false, "apply safe changes") + if e := fs.Parse(args[1:]); e != nil { + return e + } + result, e := svc.Reconcile(ctx, *apply) + _ = json.NewEncoder(os.Stdout).Encode(result) + return e + case "restore": + fs := flag.NewFlagSet("restore", flag.ContinueOnError) + id := fs.String("id", "", "attachment id") + source := fs.String("source", "", "original file") + if e := fs.Parse(args[1:]); e != nil { + return e + } + uid, e := uuid.Parse(*id) + if e != nil { + return e + } + if e = svc.Restore(ctx, uid, *source); e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"restored": uid}) + case "seed": + fs := flag.NewFlagSet("seed", flag.ContinueOnError) + count := fs.Int("count", 1, "synthetic orphan count") + keyOverride := fs.String("key", "", "explicit key for a single synthetic object") + if e := fs.Parse(args[1:]); e != nil { + return e + } + if *count < 1 || *count > 10000 { + return fmt.Errorf("count must be 1..10000") + } + keys := []string{} + for i := 0; i < *count; i++ { + key := attachment.Prefix + uuid.NewString() + if *keyOverride != "" { + if *count != 1 { + return fmt.Errorf("explicit key requires count=1") + } + key = *keyOverride + } + if e := store.Put(ctx, key, "image/png", strings.NewReader("synthetic-orphan"), 16); e != nil { + return e + } + if *count == 1 { + keys = append(keys, key) + } + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"seeded": *count, "keys": keys}) + case "remove-object": + fs := flag.NewFlagSet("remove-object", flag.ContinueOnError) + id := fs.String("id", "", "synthetic attachment id") + if e := fs.Parse(args[1:]); e != nil { + return e + } + uid, e := uuid.Parse(*id) + if e != nil { + return e + } + a, e := svc.Repo.Get(ctx, uid) + if e != nil { + return e + } + conn, unlock, e := svc.Repo.Lock(ctx, a.Key) + _ = conn + if e != nil { + return e + } + defer unlock() + if e = store.Delete(ctx, a.Key); e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"removed_object": uid}) + default: + return fmt.Errorf("unknown storage command") + } +} diff --git a/service/cmd/benchmark/main.go b/service/cmd/benchmark/main.go new file mode 100644 index 0000000..80efdf8 --- /dev/null +++ b/service/cmd/benchmark/main.go @@ -0,0 +1,260 @@ +// Command benchmark generates paced lab5 traffic through the public load balancer. +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "math" + "net/http" + "os" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/google/uuid" +) + +type client struct { + base, token string + http *http.Client +} + +func (c *client) call(method, path string, body any, key string) (map[string]any, int, error) { + var data io.Reader + if body != nil { + b, e := json.Marshal(body) + if e != nil { + return nil, 0, e + } + data = bytes.NewReader(b) + } + req, e := http.NewRequest(method, c.base+path, data) + if e != nil { + return nil, 0, e + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + if key != "" { + req.Header.Set("Idempotency-Key", key) + } + // net/http otherwise treats Idempotency-Key + GetBody as permission to retry POST. + if method == http.MethodPost { + req.GetBody = nil + } + resp, e := c.http.Do(req) + if e != nil { + return nil, 0, e + } + defer resp.Body.Close() + b, e := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if e != nil { + return nil, resp.StatusCode, e + } + if resp.StatusCode != 200 && resp.StatusCode != 201 { + return nil, resp.StatusCode, fmt.Errorf("HTTP %d", resp.StatusCode) + } + var out map[string]any + e = json.Unmarshal(b, &out) + return out, resp.StatusCode, e +} +func (c *client) setup(seed int) (string, error) { + username := "bench_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20] + password := "Synthetic-benchmark-123" + user, _, e := c.call("POST", "/api/v1/users", map[string]string{"username": username, "display_name": "Benchmark", "password": password}, "") + if e != nil { + return "", e + } + req, _ := http.NewRequest("POST", c.base+"/api/v1/auth/sessions", nil) + req.SetBasicAuth(username, password) + resp, e := c.http.Do(req) + if e != nil { + return "", e + } + var session struct { + Token string `json:"token"` + } + e = json.NewDecoder(resp.Body).Decode(&session) + resp.Body.Close() + if e != nil || resp.StatusCode != 201 || session.Token == "" { + return "", fmt.Errorf("login status %d: %v", resp.StatusCode, e) + } + c.token = session.Token + chat, _, e := c.call("POST", "/api/v1/chats", map[string]any{"title": "Benchmark", "member_ids": []any{user["id"]}}, "") + if e != nil { + return "", e + } + path := "/api/v1/chats/" + chat["id"].(string) + "/messages" + // Bounded concurrent setup is excluded from the measured interval. + var group sync.WaitGroup + slots := make(chan struct{}, 8) + errs := make(chan error, seed) + for i := range seed { + slots <- struct{}{} + group.Go(func() { + defer func() { <-slots }() + _, _, e := c.call("POST", path, map[string]string{"text": fmt.Sprintf("seed-%d", i)}, uuid.NewString()) + if e != nil { + errs <- e + } + }) + } + group.Wait() + close(errs) + for e := range errs { + if e != nil { + return "", e + } + } + return path, nil +} + +type result struct { + MeasurementUnix float64 `json:"measurement_started_unix"` + Label string `json:"label"` + TargetRate int `json:"target_rps"` + Duration float64 `json:"duration_seconds"` + Elapsed float64 `json:"elapsed_including_drain_seconds"` + Scheduled int `json:"scheduled"` + Dropped int `json:"generator_dropped"` + Success int `json:"successful_requests"` + Errors int `json:"failed_requests"` + ReadSuccess int `json:"read_success"` + WriteSuccess int `json:"write_success"` + SuccessfulRPS float64 `json:"successful_rps"` + ErrorRatio float64 `json:"error_ratio_including_drops"` + Latency map[string]float64 `json:"successful_latency_ms"` + Statuses map[int]int `json:"statuses"` + PeakInFlight int64 `json:"peak_in_flight"` + Note string `json:"note"` +} + +func phase(c *client, path string, rate int, duration time.Duration, maxInFlight int, label string) result { + out := result{Label: label, TargetRate: rate, Duration: duration.Seconds(), Statuses: map[int]int{}, Latency: map[string]float64{}, Note: "Paced open-loop, 10ms dispatch batches; 90% first-page GET, 10% POST; no POST retries. Percentiles of successes include scheduler delay; failures and capacity drops reported separately. RPS denominator includes drain."} + start := time.Now() + out.MeasurementUnix = float64(start.UnixNano()) / 1e9 + end := start.Add(duration) + slots := make(chan struct{}, maxInFlight) + var group sync.WaitGroup + var mu sync.Mutex + var active atomic.Int64 + var peak atomic.Int64 + latencies := make([]float64, 0, rate*int(duration.Seconds())) + index := 0 + for tick := 0; ; tick++ { + now := time.Now() + if !now.Before(end) { + break + } + target := start.Add(time.Duration(tick) * 10 * time.Millisecond) + if target.After(now) { + time.Sleep(time.Until(target)) + } + until := min(int(time.Since(start).Seconds()*float64(rate))+1, int(duration.Seconds()*float64(rate))) + for index < until { + i := index + index++ + out.Scheduled++ + scheduled := start.Add(time.Duration(float64(i) / float64(rate) * float64(time.Second))) + select { + case slots <- struct{}{}: + group.Go(func() { + n := active.Add(1) + for old := peak.Load(); n > old; old = peak.Load() { + if peak.CompareAndSwap(old, n) { + break + } + } + defer func() { active.Add(-1); <-slots }() + method, p := "GET", path+"?limit=50" + var body any + key := "" + if i%10 == 0 { + method = "POST" + p = path + key = uuid.NewString() + body = map[string]string{"text": key} + } + _, status, err := c.call(method, p, body, key) + latency := float64(time.Since(scheduled).Microseconds()) / 1000 + mu.Lock() + defer mu.Unlock() + out.Statuses[status]++ + if err != nil { + out.Errors++ + } else { + out.Success++ + if method == "GET" { + out.ReadSuccess++ + } else { + out.WriteSuccess++ + } + latencies = append(latencies, max(0, latency)) + } + }) + default: + out.Dropped++ + } + } + } + group.Wait() + out.Elapsed = time.Since(start).Seconds() + out.PeakInFlight = peak.Load() + out.SuccessfulRPS = float64(out.Success) / out.Elapsed + if out.Scheduled > 0 { + out.ErrorRatio = float64(out.Errors+out.Dropped) / float64(out.Scheduled) + } + slices.Sort(latencies) + for name, p := range map[string]float64{"p50": .5, "p95": .95, "p99": .99} { + if len(latencies) > 0 { + out.Latency[name] = latencies[max(0, int(math.Ceil(p*float64(len(latencies))))-1)] + } + } + return out +} +func main() { + base := flag.String("base-url", "http://localhost:18086", "public LB URL") + rate := flag.Int("rate", 100, "scheduled requests per second") + duration := flag.Duration("duration", 30*time.Second, "measured time") + warmup := flag.Duration("warmup", 5*time.Second, "warmup excluded from results") + seed := flag.Int("seed", 1000, "initial messages") + maxFlight := flag.Int("max-in-flight", 256, "bounded in-flight limit") + label := flag.String("label", "manual", "case label") + output := flag.String("output", "", "JSON destination") + flag.Parse() + if *rate < 1 || *rate > 100000 || *duration <= 0 || *warmup < 0 || *maxFlight < 1 || *seed < 0 { + fmt.Fprintln(os.Stderr, "invalid benchmark settings") + os.Exit(2) + } + c := &client{base: *base, http: &http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{MaxIdleConns: 512, MaxIdleConnsPerHost: 512, MaxConnsPerHost: *maxFlight, IdleConnTimeout: 60 * time.Second}}} + path, e := c.setup(*seed) + if e != nil { + fmt.Fprintln(os.Stderr, "setup:", e) + os.Exit(1) + } + if *warmup > 0 { + phase(c, path, *rate, *warmup, *maxFlight, "warmup") + } + row := phase(c, path, *rate, *duration, *maxFlight, *label) + b, e := json.MarshalIndent(row, "", " ") + if e != nil { + panic(e) + } + if *output != "" { + if e = os.WriteFile(*output, append(b, '\n'), 0644); e != nil { + panic(e) + } + } + fmt.Println(string(b)) + if row.Success == 0 { + os.Exit(1) + } +} diff --git a/service/cmd/benchmark/main_test.go b/service/cmd/benchmark/main_test.go new file mode 100644 index 0000000..63e1034 --- /dev/null +++ b/service/cmd/benchmark/main_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +func TestLostPostResponseIsNotRetried(t *testing.T) { + var posts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + posts.Add(1) + _, _ = io.Copy(io.Discard, r.Body) + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + _ = conn.Close() + })) + defer server.Close() + transport := &http.Transport{MaxIdleConnsPerHost: 1} + defer transport.CloseIdleConnections() + c := &client{base: server.URL, http: &http.Client{Timeout: time.Second, Transport: transport}} + if _, _, err := c.call(http.MethodGet, "/warm", nil, ""); err != nil { + t.Fatal(err) + } + if _, _, err := c.call(http.MethodPost, "/message", map[string]string{"text": "uncertain result"}, "same-key"); err == nil { + t.Fatal("lost response must be reported as an error") + } + if posts.Load() != 1 { + t.Fatalf("transport sent %d POSTs, want exactly 1", posts.Load()) + } +} diff --git a/service/go.mod b/service/go.mod new file mode 100644 index 0000000..e8220a2 --- /dev/null +++ b/service/go.mod @@ -0,0 +1,96 @@ +module gitea.privcloud.tech/mtusi2026 + +go 1.27.0 + +require ( + github.com/aws/aws-sdk-go-v2 v1.47.8 + github.com/aws/aws-sdk-go-v2/credentials v1.20.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.9 + github.com/aws/smithy-go v1.28.8 + github.com/go-chi/chi/v5 v5.5.0 + github.com/go-playground/validator/v10 v10.30.5 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/google/uuid v1.6.0 + github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 + github.com/jackc/pgx/v5 v5.11.0 + github.com/jackc/tern/v2 v2.4.3 + github.com/oapi-codegen/runtime v1.6.0 + github.com/rabbitmq/amqp091-go v1.10.0 + github.com/redis/go-redis/v9 v9.23.0 + github.com/shirou/gopsutil/v4 v4.26.10 + github.com/spf13/viper v1.21.0 + github.com/vgarvardt/pgx-google-uuid/v5 v5.6.0 + go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 + go.opentelemetry.io/otel/metric v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/sdk/log v1.47.0 + go.opentelemetry.io/otel/sdk/metric v1.47.0 + go.opentelemetry.io/otel/trace v1.47.0 + go.uber.org/multierr v1.11.0 + golang.org/x/crypto v0.57.0 + golang.org/x/sync v0.23.0 + google.golang.org/grpc v1.85.0 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.5.0 // indirect + github.com/Masterminds/sprig/v3 v3.3.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.11 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/ebitengine/purego v0.11.2 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.15 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/leodido/go-urn v1.5.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 // indirect + github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.uber.org/atomic v1.12.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/service/go.sum b/service/go.sum new file mode 100644 index 0000000..577cb02 --- /dev/null +++ b/service/go.sum @@ -0,0 +1,218 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= +github.com/aws/aws-sdk-go-v2 v1.47.8 h1:fuClo7Un+Bi0EDTnSIPWxVEoJcze+wkGzNA4Vo3jRcQ= +github.com/aws/aws-sdk-go-v2 v1.47.8/go.mod h1:I8lFSH4MGrZiw2m7miCrRYHsgpC1+HW8uiVGOiPYPcw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26 h1:SJNA4XaM8Z09adeTHf/vME93jzl44lJV70pSawIBXac= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26/go.mod h1:l0ME5pd2QytH8UZzETcrt90x9Bm+3TyBVeWfVEcZ82c= +github.com/aws/aws-sdk-go-v2/credentials v1.20.16 h1:cqRWTY4fcctJGqZedMC1Khu4h2VsgRODUOIZwn39n3Y= +github.com/aws/aws-sdk-go-v2/credentials v1.20.16/go.mod h1:x8JgCOAnbnEVQOLVX6lOtsyTrG2EHgiRNqzku9jiVxA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.11 h1:LTxHSJ/rwTFfaXqi5mlKKHJOW+shFB+HQ7NrCa4OQ48= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.11/go.mod h1:GF0bVKZVQX4UeUtPmTuWZj3gw0eRCdkVhW0xFnSRLmU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.11 h1:DGUM0yztI3Zz5kePulT/mMP/gVZFGTTISUiCcqSQtT0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.11/go.mod h1:kbcdJgN9cgSzZlUqsMveDl9IfexJqOzCihXjQXTNaPk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.11 h1:kOHCwgWFyfrz9GqnJct8knQw7Apyl3xHH+OGnome77s= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.11/go.mod h1:854cZkT2fdkdrzQklD49uzxkZnajXoelpVNgT/yk4J0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24 h1:jGzHjc2jDx08Rji1JTBtMxuXUDG7muTEw+ExYXqFzPg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24/go.mod h1:uJt75pK2cEyY80F5C+DK9YkCkJWsMkJViHxLYZRHl8Y= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.12 h1:hfO8FblXX93ZKHXSCqCjehwul1EtNJa9G8iFSOiJ+6w= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.12/go.mod h1:r8Eypc7ZRNdS7BsaLBUUdcXIEuKE6cVHZJ29u+IGaIc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.11 h1:PO7Tx443ecMI00nIxDrT99DVcbeMBQ6BUtJlDMRnY24= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.11/go.mod h1:n/9RixikO12YvNomZLP8k4TgUSoEkT1W23QIrpFAPvw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.11 h1:jNP7wZylDeSJhjc1nmjZBuG5deotp3cEYIfMazmzp+Y= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.11/go.mod h1:1US+8JBnF0xNm6GE6wM5u5ugFwSZpkd2686gMuql5tI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.9 h1:MLHGyNy4melTyk9e63s+9QgLKCHPDJrP+1NfJhtU1g4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.9/go.mod h1:zfUkiNK2JiYlR1/SnU8aJ8Gj1nMFLkPZ69rVLv6jWPo= +github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ= +github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ebitengine/purego v0.11.2 h1:WWiTx8rD8zYdOwSj8O/NiNbsOvSAgJMTlyV1XlbUSeU= +github.com/ebitengine/purego v0.11.2/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI= +github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ= +github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg= +github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.5 h1:YyCXvVShZbs2Sm3Mb53eNOlhRXctSOzW5QJAouCTZL4= +github.com/go-playground/validator/v10 v10.30.5/go.mod h1:wEqiaov48pXX1kjhc3Da8y0M0Dtg/BK7gurFBLgwFrQ= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 h1:D/V0gu4zQ3cL2WKeVNVM4r2gLxGGf6McLwgXzRTo2RQ= +github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/tern/v2 v2.4.3 h1:g293d3OZgW7OFEhsYXgEv0C21jea2boNr0VR4K5I7OY= +github.com/jackc/tern/v2 v2.4.3/go.mod h1:rMpMuRYcff5wWLptoTSO1qcDxJ4OodysvK17i2SVBys= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.5.0 h1:pLqT2kq1zpHW/1D18QMjMpdtX7cekxqtJJjg5ANyWw0= +github.com/leodido/go-urn v1.5.0/go.mod h1:9BORnCDhdPBJNDEX+w1bJisa8yOKYi116VeO96s4ifE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/oapi-codegen/runtime v1.6.0 h1:7Xx+GlueD6nRuyKoCPzL434Jfi3BetbiJOrzCHp/VPU= +github.com/oapi-codegen/runtime v1.6.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= +github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 h1:gIZQQ2+wbsdQGWLPDSdJFptR25tBCjSC0KbSIQZ1rgE= +github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU= +github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= +github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/shirou/gopsutil/v4 v4.26.10 h1:MJN6euesJx53MVEjscYeyRSRcrdIcy3sz5k9nosjldg= +github.com/shirou/gopsutil/v4 v4.26.10/go.mod h1:XpkIrcUMQhwte/dGxu57KtnMMFsbKelI7PP23zweKBk= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/vgarvardt/pgx-google-uuid/v5 v5.6.0 h1:EhPtK0mgrgaTMXpegE69hvoSOVC1Ahk8+QJ9B8b+OdU= +github.com/vgarvardt/pgx-google-uuid/v5 v5.6.0/go.mod h1:5LtFrNEkgzxHvXPO9eOvcXsSn9/KeKYgx9kjeI2oXQI= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNRaZy4v4YgVenGd52dRBI= +go.opentelemetry.io/contrib/bridges/otelslog v0.21.0/go.mod h1:9pBdbVbuehIDzxN4accnzzxEUPT8Nq7grbZMo+J1VN4= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 h1:5WVp40k2fJHuc5BtNTgKk+EugKFT8Eg3lywFfCPI/nE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0/go.mod h1:59nH0RYCGHRlwt15WyioeSYINBbuObeBpTowPhEKWHY= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 h1:7kszRN4Ze5GeAkUnBjBSUIunVBMA+mZIG04auueqguM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0/go.mod h1:CLzakYy8jOdkBfcveCWYkeKJxujcF4xPhh2Z7IfE98U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/log v1.47.0 h1:W4N/ZfgpkRRnmARnqYoQ6BrWA0mrj8IzVDNLlpmUJZY= +go.opentelemetry.io/otel/sdk/log v1.47.0/go.mod h1:Wu74Z2EmOKXi2FleSNAb0Hyau2G+r5kubSEMJw5aAMs= +go.opentelemetry.io/otel/sdk/log/logtest v0.23.0 h1:j3pm42QCrKYYLLkI6OrV1QHFkUfDA6/4EP+Y1PzggLg= +go.opentelemetry.io/otel/sdk/log/logtest v0.23.0/go.mod h1:zt0gEnuTZgDvqvczHwYswz93eqMaIr6upTTsdV/HUlU= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= +go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw= +google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/service/internal/adapter/broker/rabbitmq.go b/service/internal/adapter/broker/rabbitmq.go new file mode 100644 index 0000000..f62d9f2 --- /dev/null +++ b/service/internal/adapter/broker/rabbitmq.go @@ -0,0 +1,194 @@ +package broker + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/fault" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + amqp "github.com/rabbitmq/amqp091-go" + "go.opentelemetry.io/otel/trace" + "io" + "log/slog" + "net" + "regexp" + "time" +) + +const Queue = "attachments.v1" + +var tracePattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) + +type Rabbit struct{ URL string } + +func (r *Rabbit) connect(deadline time.Duration) (*amqp.Connection, *amqp.Channel, error) { + dial := amqp.DefaultDial(5 * time.Second) + if deadline > 0 { + dial = func(network, address string) (net.Conn, error) { + c, e := net.DialTimeout(network, address, 5*time.Second) + if e != nil { + return nil, e + } + if e = c.SetDeadline(time.Now().Add(deadline)); e != nil { + c.Close() + return nil, e + } + return c, nil + } + } + conn, e := amqp.DialConfig(r.URL, amqp.Config{Heartbeat: 10 * time.Second, Dial: dial}) + if e != nil { + return nil, nil, errors.New("broker connection unavailable") + } + ch, e := conn.Channel() + if e != nil { + conn.Close() + return nil, nil, errors.New("broker channel unavailable") + } + _, e = ch.QueueDeclare(Queue+".invalid", true, false, false, false, nil) + if e == nil { + e = ch.ExchangeDeclare("attachments.invalid", "direct", true, false, false, false, nil) + } + if e == nil { + e = ch.QueueBind(Queue+".invalid", "invalid", "attachments.invalid", false, nil) + } + if e == nil { + _, e = ch.QueueDeclare(Queue, true, false, false, false, amqp.Table{"x-dead-letter-exchange": "attachments.invalid", "x-dead-letter-routing-key": "invalid"}) + } + if e != nil { + ch.Close() + conn.Close() + return nil, nil, errors.New("broker topology unavailable") + } + return conn, ch, nil +} +func (r *Rabbit) Publish(ctx context.Context, m outbox.DeliveryMessage) (err error) { + var job dto.JobMessage + if e := json.Unmarshal(m.Payload, &job); e != nil { + return e + } + ctx, sp := telemetry.Start(telemetry.Extract(ctx, job.Traceparent, job.Tracestate), "attachments publish", trace.SpanKindProducer) + defer func() { telemetry.End(sp, err) }() + job.Traceparent, job.Tracestate = telemetry.Inject(ctx) + body, marshalErr := json.Marshal(job) + if marshalErr != nil { + return marshalErr + } + conn, ch, e := r.connect(20 * time.Second) + if e != nil { + return e + } + defer conn.Close() + defer ch.Close() + if e = ch.Confirm(false); e != nil { + return e + } + confirms := ch.NotifyPublish(make(chan amqp.Confirmation, 1)) + returns := ch.NotifyReturn(make(chan amqp.Return, 1)) + fault.Hit(ctx, "before_publish", job.JobID) + if e = ch.PublishWithContext(ctx, "", Queue, true, false, amqp.Publishing{DeliveryMode: amqp.Persistent, ContentType: "application/json", MessageId: m.ID.String(), Body: body}); e != nil { + return errors.New("broker publish failed") + } + select { + case c, ok := <-confirms: + if !ok || !c.Ack { + return errors.New("broker did not confirm") + } + select { + case <-returns: + return errors.New("unroutable job") + default: + } + case <-time.After(15 * time.Second): + return errors.New("confirm timeout") + case <-ctx.Done(): + return ctx.Err() + } + fault.Hit(ctx, "after_publish", job.JobID) + return nil +} +func (r *Rabbit) Consume(ctx context.Context, concurrency, prefetch int, process func(context.Context, dto.JobMessage) error) error { + conn, ch, e := r.connect(0) + if e != nil { + return e + } + defer conn.Close() + defer ch.Close() + if e = ch.Qos(prefetch, 0, false); e != nil { + return e + } + deliveries, e := ch.Consume(Queue, "worker-"+uuid.NewString(), false, false, false, false, nil) + if e != nil { + return e + } + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + conn.Close() + case <-done: + } + }() + defer close(done) + tasks := make(chan amqp.Delivery) + finished := make(chan struct{}, concurrency) + for range concurrency { + go func() { + defer func() { finished <- struct{}{} }() + for d := range tasks { + var m dto.JobMessage + dec := json.NewDecoder(bytes.NewReader(d.Body)) + dec.DisallowUnknownFields() + e := dec.Decode(&m) + if e != nil || dec.Decode(new(any)) != io.EOF || m.SchemaVersion != 1 || m.Type != "attachment.process.v1" || m.JobID == uuid.Nil || m.AttachmentID == uuid.Nil || m.Attempt < 1 || m.PipelineVersion < 1 || m.CreatedAt.IsZero() || len(m.Tracestate) > 512 || (m.Traceparent != "" && !tracePattern.MatchString(m.Traceparent)) { + _ = d.Reject(false) + continue + } + jobctx, sp := telemetry.Start(telemetry.Extract(ctx, m.Traceparent, m.Tracestate), "attachments consume", trace.SpanKindConsumer) + slog.InfoContext(jobctx, "job_received", "job_id", m.JobID, "attempt", m.Attempt) + e = process(jobctx, m) + telemetry.End(sp, e) + if e != nil { + slog.WarnContext(jobctx, "job_not_acknowledged", "job_id", m.JobID, "reason", "processing_or_dependency_unavailable") + select { + case <-ctx.Done(): + case <-time.After(200 * time.Millisecond): + } + _ = d.Nack(false, true) + continue + } + fault.Hit(ctx, "before_ack", m.JobID) + _ = d.Ack(false) + } + }() + } + for d := range deliveries { + select { + case tasks <- d: + case <-ctx.Done(): + } + } + close(tasks) + for range concurrency { + <-finished + } + if ctx.Err() != nil { + return ctx.Err() + } + return errors.New("broker consumer disconnected") +} + +func (r *Rabbit) Depth(ctx context.Context) (int, error) { + conn, ch, e := r.connect(2 * time.Second) + if e != nil { + return 0, e + } + defer conn.Close() + defer ch.Close() + q, e := ch.QueueInspect(Queue) + return q.Messages, e +} diff --git a/service/internal/adapter/controller/http/attachment.go b/service/internal/adapter/controller/http/attachment.go new file mode 100644 index 0000000..d02cfda --- /dev/null +++ b/service/internal/adapter/controller/http/attachment.go @@ -0,0 +1 @@ +package http diff --git a/service/internal/adapter/controller/http/attachmentHandler.go b/service/internal/adapter/controller/http/attachmentHandler.go new file mode 100644 index 0000000..d286d53 --- /dev/null +++ b/service/internal/adapter/controller/http/attachmentHandler.go @@ -0,0 +1,111 @@ +package http + +import ( + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "io" + "mime" + "net/http" + "strconv" + "strings" +) + +type AttachmentHandler struct { + router router.Router + provider jwtprovider.Provider + svc *attachment.Service +} + +func NewAttachmentHandler(r router.Router, p jwtprovider.Provider, s *attachment.Service) Handler { + return &AttachmentHandler{r, p, s} +} +func (h *AttachmentHandler) RegisterRoutes() { + h.router.Private("POST", "/api/v1/chats/{chat_id}/attachments", h.Upload) + h.router.Private("GET", "/api/v1/attachments/{attachment_id}", h.Get) + h.router.Private("GET", "/api/v1/attachments/{attachment_id}/content", h.Get) +} +func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) { + user, e := framework.ParseToken(r, h.provider) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + id, e := framework.ParseIDFromPathParam(r.Context(), w, r, "chat_id") + if e != nil { + return + } + values := r.URL.Query()["filename"] + if len(values) != 1 { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "expected one filename")) + return + } + if !attachment.ValidFilename(values[0]) { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid filename")) + return + } + mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type")) + if e != nil || (mt != "image/png" && mt != "image/jpeg") { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG")) + return + } + if r.ContentLength > h.svc.Config.MaxBytes { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodePayloadTooLarge, "upload too large")) + return + } + a, e := h.svc.Upload(r.Context(), dto.UploadAttachment{UserID: user.UserID, ChatID: id, Filename: values[0], ContentType: mt, Body: io.LimitReader(r.Body, h.svc.Config.MaxBytes+1)}) + if e != nil { + if apperr.CodeFromError(e) == apperr.CodeTooManyRequests { + w.Header().Set("Retry-After", "1") + } + framework.JSONError(r.Context(), w, e) + return + } + framework.JSONResponse(w, http.StatusAccepted, a) +} +func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) { + user, e := framework.ParseToken(r, h.provider) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + id, e := framework.ParseIDFromPathParam(r.Context(), w, r, "attachment_id") + if e != nil { + return + } + + if !strings.HasSuffix(r.URL.Path, "/content") { + a, e := h.svc.Get(r.Context(), user.UserID, id) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + framework.JSONResponse(w, 200, a.Public) + return + } + v := r.URL.Query().Get("variant") + if v == "" { + v = "original" + } + if v != "original" && v != "thumbnail" && v != "crop" && v != "optimized" { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant")) + return + } + obj, e := h.svc.Content(r.Context(), user.UserID, id, v) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + defer obj.Body.Close() + w.Header().Set("Content-Type", obj.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10)) + w.Header().Set("X-Content-Type-Options", "nosniff") + // Once streaming has started a transport failure terminates the response; no JSON + // error can be appended to an image. Content-Length exposes incomplete downloads. + if _, e = io.Copy(w, obj.Body); e != nil { + panic(http.ErrAbortHandler) + } +} diff --git a/service/internal/adapter/controller/http/auth.go b/service/internal/adapter/controller/http/auth.go new file mode 100644 index 0000000..73b263c --- /dev/null +++ b/service/internal/adapter/controller/http/auth.go @@ -0,0 +1,88 @@ +package http + +import ( + "net/http" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/service/auth" +) + +type AuthHandler struct { + router router.Router + + auth auth.API +} + +func NewAuthHandler(r router.Router, api auth.API) Handler { + return &AuthHandler{ + router: r, + auth: api, + } +} + +func (h *AuthHandler) RegisterRoutes() { + h.router.Public(http.MethodPost, "/api/v1/auth/sessions", h.Login) + h.router.Private(http.MethodDelete, "/api/v1/auth/sessions/current", h.Logout) +} + +func (h *AuthHandler) Login(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + username, password, ok := r.BasicAuth() + if !ok || username == "" || password == "" { + w.Header().Set( + "WWW-Authenticate", + `Basic realm="messenger", charset="UTF-8"`, + ) + framework.JSONError(ctx, w, apperr.New(apperr.CodeUnauthorized, "invalid credentials")) + return + } + + resp, err := h.auth.Login(ctx, dto.Login{ + Username: username, + Password: password, + }) + if err != nil { + if apperr.HTTPStatusFromError(err) == http.StatusUnauthorized { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + } + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusCreated, contracts.Session{ + ExpiresAt: resp.ExpiresAt, + Token: resp.Tokens.RefreshToken.Payload, + TokenType: resp.TokenType, + User: contracts.User{ + CreatedAt: resp.User.CreatedAt, + DisplayName: resp.User.DisplayName, + Id: resp.User.ID, + Username: resp.User.Username, + }, + }) +} + +func (h *AuthHandler) Logout(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + token, exists := framework.Token(r) + if !exists { + framework.JSONError(ctx, w, apperr.New(apperr.CodeUnauthorized, "missing token in header")) + return + } + + err := h.auth.Logout(ctx, dto.RefreshToken{ + Payload: token, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusNoContent, nil) +} diff --git a/service/internal/adapter/controller/http/chat.go b/service/internal/adapter/controller/http/chat.go new file mode 100644 index 0000000..dcf583d --- /dev/null +++ b/service/internal/adapter/controller/http/chat.go @@ -0,0 +1,287 @@ +package http + +import ( + "bytes" + "encoding/json" + "net/http" + "regexp" + "strings" + "unicode/utf8" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/pkg/validator" + "gitea.privcloud.tech/mtusi2026/internal/service/chat" +) + +var messageKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +type ChatHandler struct { + router router.Router + provider jwtprovider.Provider + validator *validator.ValidatorImpl + + chat chat.API +} + +func NewChatHandler(r router.Router, provider jwtprovider.Provider, v *validator.ValidatorImpl, api chat.API) Handler { + return &ChatHandler{ + router: r, + provider: provider, + validator: v, + chat: api, + } +} + +func (h *ChatHandler) RegisterRoutes() { + h.router.Private(http.MethodPost, "/api/v1/chats", h.Create) + h.router.Private(http.MethodGet, "/api/v1/chats", h.Get) + h.router.Private(http.MethodGet, "/api/v1/chats/{chat_id}", h.GetByID) + h.router.Private(http.MethodPost, "/api/v1/chats/{chat_id}/messages", h.SendMessage) + h.router.Private(http.MethodGet, "/api/v1/chats/{chat_id}/messages", h.GetMessages) +} + +func (h *ChatHandler) Create(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + var req contracts.CreateChat + if !framework.Decode(w, r, &req) || !framework.Validate(w, r, h.validator, &req) { + return + } + + if userData.UserID == uuid.Nil || + !validText(req.Title, 100) || + len(req.MemberIds) < 1 || + len(req.MemberIds) > 100 { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid chat fields")) + return + } + + resp, err := h.chat.CreateChat(ctx, dto.CreateChat{ + CreatorID: userData.UserID, + Title: req.Title, + MemberIDs: req.MemberIds, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusCreated, contracts.Chat{ + Id: resp.ID, + Title: resp.Title, + MemberIds: resp.MemberIDs, + CreatedAt: resp.CreatedAt, + }) +} + +func (h *ChatHandler) Get(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + limit, afterID, ok := framework.PageParams(w, r) + if !ok { + return + } + + if userData.UserID == uuid.Nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid user id")) + return + } + if afterID != nil && *afterID == uuid.Nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid cursor")) + return + } + + page, err := h.chat.ListForMember(ctx, dto.ChatFilters{ + UserID: userData.UserID, + Limit: limit, + AfterID: afterID, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + items := make([]contracts.Chat, 0, len(page.Items)) + for _, item := range page.Items { + items = append(items, contracts.Chat{ + CreatedAt: item.CreatedAt, + Id: item.ID, + MemberIds: item.MemberIDs, + Title: item.Title, + }) + } + + framework.JSONResponse(w, http.StatusOK, contracts.ChatPage{ + Items: items, + NextCursor: cursorString(page.NextCursorID), + }) +} + +func (h *ChatHandler) GetByID(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + chatID, err := framework.ParseIDFromPathParam(ctx, w, r, "chat_id") + if err != nil { + return + } + + resp, err := h.chat.GetForMember(ctx, dto.ChatMember{ + UserID: userData.UserID, + ChatID: chatID, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusOK, contracts.Chat{ + CreatedAt: resp.CreatedAt, + Id: resp.ID, + MemberIds: resp.MemberIDs, + Title: resp.Title, + }) +} + +func (h *ChatHandler) SendMessage(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + chatID, err := framework.ParseIDFromPathParam(ctx, w, r, "chat_id") + if err != nil { + return + } + + keyValues := r.Header.Values("Idempotency-Key") + if len(keyValues) != 1 || !messageKeyPattern.MatchString(keyValues[0]) { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid idempotency key")) + return + } + + var req struct { + Text *string `json:"text"` + AttachmentIDs json.RawMessage `json:"attachment_ids"` + } + if !framework.Decode(w, r, &req) || !framework.Validate(w, r, h.validator, &req) { + return + } + + if req.Text == nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "text is required")) + return + } + + ids := []uuid.UUID{} + if len(req.AttachmentIDs) > 0 { + if bytes.Equal(bytes.TrimSpace(req.AttachmentIDs), []byte("null")) || json.Unmarshal(req.AttachmentIDs, &ids) != nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid attachment_ids")) + return + } + } + + message, err := h.chat.CreateMessage(ctx, dto.CreateMessage{ + IdempotencyKey: keyValues[0], + ChatID: chatID, + SenderID: userData.UserID, + Text: *req.Text, + AttachmentIDs: ids, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusCreated, contracts.Message{ + Id: message.ID, + ChatId: message.ChatID, + SenderId: message.SenderID, + Text: message.Text, + CreatedAt: message.CreatedAt, + AttachmentIds: message.AttachmentIDs, + }) +} + +func (h *ChatHandler) GetMessages(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + chatID, err := framework.ParseIDFromPathParam(ctx, w, r, "chat_id") + if err != nil { + return + } + + limit, afterID, ok := framework.PageParams(w, r) + if !ok { + return + } + + page, err := h.chat.ListForChat(ctx, dto.MessageFilters{ + ChatID: chatID, + UserID: userData.UserID, + Limit: limit, + AfterID: afterID, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + items := make([]contracts.Message, 0, len(page.Items)) + for _, message := range page.Items { + items = append(items, contracts.Message{ + Id: message.ID, + ChatId: message.ChatID, + SenderId: message.SenderID, + Text: message.Text, + CreatedAt: message.CreatedAt, + AttachmentIds: message.AttachmentIDs, + }) + } + framework.JSONResponse(w, http.StatusOK, contracts.MessagePage{Items: items, NextCursor: cursorString(page.NextCursorID)}) +} + +func cursorString(id *uuid.UUID) *string { + if id == nil { + return nil + } + value := id.String() + return &value +} + +func validText(value string, maxLength int) bool { + return utf8.ValidString(value) && strings.TrimSpace(value) != "" && utf8.RuneCountInString(value) <= maxLength +} diff --git a/service/internal/adapter/controller/http/controller.go b/service/internal/adapter/controller/http/controller.go new file mode 100644 index 0000000..47178ce --- /dev/null +++ b/service/internal/adapter/controller/http/controller.go @@ -0,0 +1,102 @@ +package http + +import ( + "context" + "errors" + "log/slog" + "net" + "net/http" + "sync/atomic" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" +) + +type Server interface { + ShouldBeRunning() bool + Ready() bool + Start(ctx context.Context) error + Stop(ctx context.Context) error +} + +type server struct { + server http.Server + cfg *config.Server + ready atomic.Bool +} + +func New(cfg *config.Config, router router.Router) Server { + s := server{ + server: http.Server{ + Addr: cfg.Server.BindAddress(), + Handler: router, + ReadTimeout: cfg.Server.ReadTimeout, + WriteTimeout: cfg.Server.WriteTimeout, + IdleTimeout: cfg.Server.IdleTimeout, + }, + cfg: cfg.Server, + ready: atomic.Bool{}, + } + + if err := s.validate(); err != nil { + panic(err) + } + + return &s +} + +func (s *server) Start(_ context.Context) error { + if !s.ShouldBeRunning() { + slog.Info("server should not be running: skip starting") + return nil + } + listener, err := net.Listen("tcp", s.server.Addr) + if err != nil { + return err + } + + s.ready.Store(true) + go func() { + defer s.ready.Store(false) + if err := s.server.Serve(listener); err != nil && + !errors.Is(err, http.ErrServerClosed) { + slog.Error("rest server failed", "error", err) + } + }() + return nil +} + +func (s *server) Stop(ctx context.Context) error { + if !s.ShouldBeRunning() { + slog.Info("server should not be running: skip stopping") + return nil + } + + slog.Info("shutting down rest server...", "address", s.server.Addr) + s.ready.Store(false) + if err := s.server.Shutdown(ctx); err != nil { + slog.Error("rest server shutdown failed", "error", err) + _ = s.server.Close() + return err + } + slog.Info("rest server shutdown successfully") + return nil +} + +func (s *server) ShouldBeRunning() bool { + return s.cfg.Enabled +} + +func (s *server) Ready() bool { + return s.ready.Load() +} + +func (s *server) validate() error { + if s.cfg == nil { + slog.Error("failed to validate server: provided nil config") + return errors.New("failed to validate server: provided nil config") + } + + slog.Info("server successfully validated") + return nil +} diff --git a/service/internal/adapter/controller/http/handler.go b/service/internal/adapter/controller/http/handler.go new file mode 100644 index 0000000..275bee0 --- /dev/null +++ b/service/internal/adapter/controller/http/handler.go @@ -0,0 +1,5 @@ +package http + +type Handler interface { + RegisterRoutes() +} diff --git a/service/internal/adapter/controller/http/health.go b/service/internal/adapter/controller/http/health.go new file mode 100644 index 0000000..4d021a7 --- /dev/null +++ b/service/internal/adapter/controller/http/health.go @@ -0,0 +1,60 @@ +package http + +import ( + "context" + "net/http" + "time" + + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" +) + +type databasePinger interface{ Ping(context.Context) error } + +type HealthHandler struct { + router router.Router + server Server + database databasePinger + storage []databasePinger +} + +func NewHealthHandler(r router.Router, server Server, database databasePinger, storage ...databasePinger) Handler { + return &HealthHandler{ + router: r, + server: server, + database: database, + storage: storage, + } +} + +func (h *HealthHandler) RegisterRoutes() { + h.router.Public(http.MethodGet, "/health/ready", h.Ready) + h.router.Public(http.MethodGet, "/health/live", h.Live) +} + +func (h *HealthHandler) Live(w http.ResponseWriter, _ *http.Request) { + framework.JSONResponse(w, http.StatusOK, contracts.Health{Status: contracts.Ok}) +} + +func (h *HealthHandler) Ready(w http.ResponseWriter, r *http.Request) { + if !h.server.Ready() { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnavailable, "server is not ready")) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 200*time.Millisecond) + defer cancel() + if err := h.database.Ping(ctx); err != nil { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnavailable, "database unavailable")) + return + } + for _, dependency := range h.storage { + if err := dependency.Ping(ctx); err != nil { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnavailable, "storage unavailable")) + return + } + } + + framework.JSONResponse(w, http.StatusOK, contracts.Health{Status: contracts.Ok}) +} diff --git a/service/internal/adapter/controller/http/router.go b/service/internal/adapter/controller/http/router.go new file mode 100644 index 0000000..d02cfda --- /dev/null +++ b/service/internal/adapter/controller/http/router.go @@ -0,0 +1 @@ +package http diff --git a/service/internal/adapter/controller/http/user.go b/service/internal/adapter/controller/http/user.go new file mode 100644 index 0000000..7fdb640 --- /dev/null +++ b/service/internal/adapter/controller/http/user.go @@ -0,0 +1,113 @@ +package http + +import ( + "net/http" + "regexp" + "strings" + "unicode/utf8" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/pkg/validator" + "gitea.privcloud.tech/mtusi2026/internal/service/user" +) + +type UserHandler struct { + router router.Router + provider jwtprovider.Provider + validator *validator.ValidatorImpl + + user user.API +} + +func NewUserHandler(r router.Router, provider jwtprovider.Provider, v *validator.ValidatorImpl, api user.API) Handler { + return &UserHandler{ + router: r, + provider: provider, + validator: v, + user: api, + } +} + +var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +func (h *UserHandler) RegisterRoutes() { + h.router.Public(http.MethodPost, "/api/v1/users", h.Create) + h.router.Private(http.MethodGet, "/api/v1/users/me", h.GetMe) +} + +func (h *UserHandler) Create(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + var req contracts.CreateUser + if !framework.Decode(w, r, &req) || !framework.Validate(w, r, h.validator, &req) { + return + } + + toCreate := dto.CreateUser{ + Username: req.Username, + DisplayName: req.DisplayName, + } + + if req.Password != nil { + toCreate.Password = *req.Password + } + + if !usernamePattern.MatchString(req.Username) || !utf8.ValidString(req.DisplayName) || + strings.TrimSpace(req.DisplayName) == "" || utf8.RuneCountInString(req.DisplayName) > 100 { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid user fields")) + return + } + passwordLen := utf8.RuneCountInString(toCreate.Password) + if !utf8.ValidString(toCreate.Password) || strings.TrimSpace(toCreate.Password) == "" || passwordLen < 12 || passwordLen > 128 { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid password")) + return + } + + resp, err := h.user.Create(ctx, toCreate) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusCreated, contracts.User{ + CreatedAt: resp.CreatedAt, + DisplayName: resp.DisplayName, + Id: resp.ID, + Username: resp.Username, + }) +} + +func (h *UserHandler) GetMe(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + if userData.UserID == uuid.Nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid user id")) + return + } + + resp, err := h.user.GetByID(ctx, userData.UserID) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusOK, contracts.User{ + CreatedAt: resp.CreatedAt, + DisplayName: resp.DisplayName, + Id: resp.ID, + Username: resp.Username, + }) +} diff --git a/service/internal/adapter/repository/cache/outside/chat.go b/service/internal/adapter/repository/cache/outside/chat.go new file mode 100644 index 0000000..a5eb60c --- /dev/null +++ b/service/internal/adapter/repository/cache/outside/chat.go @@ -0,0 +1,78 @@ +package outside + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/cache" +) + +const ( + // chatRedisKey equals "chat:{chat_id}" path + chatRedisKey = "chat:%s" +) + +type ChatCache struct { + client *cache.Client +} + +func NewChatCache(client *cache.Client) *ChatCache { + return &ChatCache{ + client: client, + } +} + +type chat struct { + ID uuid.UUID `json:"id"` + Title string `json:"title"` + MemberIDs []uuid.UUID `json:"member_ids"` + CreatedAt time.Time `json:"created_at"` +} + +func (r *ChatCache) Create(ctx context.Context, req dto.Chat) error { + data, err := json.Marshal(chat{ + ID: req.ID, + Title: req.Title, + MemberIDs: req.MemberIDs, + CreatedAt: req.CreatedAt, + }) + if err != nil { + return apperr.Wrap(err, apperr.CodeInternal, "failed to marshal chat cache") + } + + ctx, cancel := context.WithTimeout(ctx, cache.OperationTimeout) + defer cancel() + err = r.client.Set(ctx, fmt.Sprintf(chatRedisKey, req.ID), data, 0).Err() + if err != nil { + return apperr.WrapRedis(err, "failed to create chat cache") + } + + return nil +} + +func (r *ChatCache) Get(ctx context.Context, chatID uuid.UUID) (*dto.Chat, error) { + ctx, cancel := context.WithTimeout(ctx, cache.OperationTimeout) + defer cancel() + data, err := r.client.Get(ctx, fmt.Sprintf(chatRedisKey, chatID.String())).Bytes() + if err != nil { + return nil, apperr.WrapRedis(err, "failed to read chat cache") + } + + var item chat + if err = json.Unmarshal(data, &item); err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to unmarshal chat cache") + } + + return &dto.Chat{ + ID: item.ID, + Title: item.Title, + MemberIDs: item.MemberIDs, + CreatedAt: item.CreatedAt, + }, nil +} diff --git a/service/internal/adapter/repository/storage/attachment_scan.go b/service/internal/adapter/repository/storage/attachment_scan.go new file mode 100644 index 0000000..5a10537 --- /dev/null +++ b/service/internal/adapter/repository/storage/attachment_scan.go @@ -0,0 +1,39 @@ +package storage + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +func (r *AttachmentsRepo) Page(ctx context.Context, after uuid.UUID) ([]*AttachmentRecord, error) { + rows, e := r.Pool.Query(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id>$1 ORDER BY id LIMIT 100", after) + if e != nil { + return nil, apperr.WrapPostgres(e, "scan failed") + } + defer rows.Close() + out := make([]*AttachmentRecord, 0, 100) + for rows.Next() { + a, e := ScanAttachment(rows) + if e != nil { + return nil, e + } + out = append(out, a) + } + return out, rows.Err() +} +func (r *AttachmentsRepo) Exists(ctx context.Context, c *pgxpool.Conn, key string) (bool, error) { + var found bool + e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachment_variants WHERE object_key=$1 UNION ALL SELECT 1 FROM attachments WHERE object_key=$1 OR staging_key=$1 OR variant_keys @> jsonb_build_object('thumbnail',$1::text) OR variant_keys @> jsonb_build_object('crop',$1::text) OR variant_keys @> jsonb_build_object('optimized',$1::text))", key).Scan(&found) + return found, e +} +func (r *AttachmentsRepo) MarkMissing(ctx context.Context, c *pgxpool.Conn, id uuid.UUID, missing bool) (bool, error) { + tag, e := c.Exec(ctx, "UPDATE attachments SET integrity_missing=$2,updated_at=clock_timestamp() WHERE id=$1 AND integrity_missing<>$2", id, missing) + return tag.RowsAffected() == 1, e +} + +func (r *AttachmentsRepo) CheckConn(ctx context.Context, c *pgxpool.Conn) error { + _, e := c.Exec(ctx, "SELECT 1") + return e +} diff --git a/service/internal/adapter/repository/storage/attachments.go b/service/internal/adapter/repository/storage/attachments.go new file mode 100644 index 0000000..0a3b2bb --- /dev/null +++ b/service/internal/adapter/repository/storage/attachments.go @@ -0,0 +1,65 @@ +package storage + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" +) + +type AttachmentRecord = dto.AttachmentRecord +type AttachmentsRepo struct{ Pool *pgxpool.Pool } + +const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing,status,error_code,pipeline_version,staging_key,variants,variant_keys" + +func ScanAttachment(row pgx.Row) (*AttachmentRecord, error) { + a := new(AttachmentRecord) + p := &a.Public + var variants, keys []byte + if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing, &p.Status, &p.ErrorCode, &a.PipelineVersion, &a.StagingKey, &variants, &keys); err != nil { + return nil, apperr.WrapPostgres(err, "attachment unavailable") + } + if err := json.Unmarshal(variants, &p.Variants); err != nil { + return nil, err + } + if err := json.Unmarshal(keys, &a.VariantKeys); err != nil { + return nil, err + } + return a, nil +} +func (r *AttachmentsRepo) Get(ctx context.Context, id uuid.UUID) (*AttachmentRecord, error) { + return ScanAttachment(r.Pool.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1", id)) +} + +// Session locks span S3 work and the database commit. A lost connection releases +// the lock; the mandatory grace period also exceeds the maximum upload lifetime. +func (r *AttachmentsRepo) Lock(ctx context.Context, key string) (*pgxpool.Conn, func(), error) { + c, e := r.Pool.Acquire(ctx) + if e != nil { + return nil, nil, apperr.WrapPostgres(e, "database unavailable") + } + if _, e = c.Exec(ctx, "SELECT pg_advisory_lock(hashtextextended($1, 42))", key); e != nil { + // The server may have acquired the session lock before cancellation was + // observed by the client. Never return that uncertain session to the pool. + clean, cancel := context.WithTimeout(context.Background(), 2*time.Second) + _ = c.Conn().Close(clean) + cancel() + c.Release() + return nil, nil, apperr.WrapPostgres(e, "lock unavailable") + } + return c, func() { + // Close rather than return a possibly locked session when unlock fails. + clean, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if _, e := c.Exec(clean, "SELECT pg_advisory_unlock(hashtextextended($1, 42))", key); e != nil { + _ = c.Conn().Close(clean) + } + c.Release() + }, nil +} diff --git a/service/internal/adapter/repository/storage/chats.go b/service/internal/adapter/repository/storage/chats.go new file mode 100644 index 0000000..15a7298 --- /dev/null +++ b/service/internal/adapter/repository/storage/chats.go @@ -0,0 +1,91 @@ +package storage + +import ( + "context" + "errors" + + "github.com/google/uuid" + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" +) + +type ChatsRepo struct { + pool *pgxpool.Pool +} + +func NewChatsRepo(pool *pgxpool.Pool) *ChatsRepo { + return &ChatsRepo{pool: pool} +} + +func (r *ChatsRepo) CreateTx(ctx context.Context, tx pgx.Tx, req dto.Chat) error { + const insertChat = `INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)` + if _, err := tx.Exec(ctx, insertChat, req.ID, req.Title, req.CreatedAt); err != nil { + return apperr.WrapPostgres(err, "failed to create chat") + } + const insertMembers = `INSERT INTO chat_members (chat_id, user_id) +SELECT $1, user_id FROM unnest($2::uuid[]) AS members(user_id)` + if _, err := tx.Exec(ctx, insertMembers, req.ID, req.MemberIDs); err != nil { + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && + pgErr.Code == pgerrcode.ForeignKeyViolation && pgErr.ConstraintName == "chat_members_user_fk" { + return apperr.Wrap(err, apperr.CodeBadRequest, "unknown chat member") + } + return apperr.WrapPostgres(err, "failed to create chat members") + } + return nil +} + +func (r *ChatsRepo) GetForMember(ctx context.Context, req dto.ChatMember) (*dto.Chat, error) { + const query = `SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) +FROM chats c +WHERE c.id = $1 + AND EXISTS (SELECT 1 FROM chat_members access WHERE access.chat_id = c.id AND access.user_id = $2)` + return scanChat(r.pool.QueryRow(ctx, query, req.ChatID, req.UserID)) +} + +func (r *ChatsRepo) ListForMember(ctx context.Context, userID uuid.UUID, page dto.PageQuery) ([]dto.Chat, error) { + query := `WITH page AS MATERIALIZED ( + SELECT c.id, c.title, c.created_at + FROM chats c JOIN chat_members access ON access.chat_id = c.id + WHERE access.user_id = $1` + args := []any{userID, page.Limit} + if page.After != nil { + query += ` AND (c.created_at, c.id) > ($3, $4)` + args = append(args, page.After.CreatedAt, page.After.ID) + } + query += ` ORDER BY c.created_at, c.id LIMIT $2 +) +SELECT p.id, p.title, p.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = p.id ORDER BY m.user_id) +FROM page p ORDER BY p.created_at, p.id` + rows, err := r.pool.Query(ctx, query, args...) + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to list chats") + } + defer rows.Close() + chats := make([]dto.Chat, 0) + for rows.Next() { + chat, err := scanChat(rows) + if err != nil { + return nil, err + } + chats = append(chats, *chat) + } + if err = rows.Err(); err != nil { + return nil, apperr.WrapPostgres(err, "failed to list chats") + } + return chats, nil +} + +func scanChat(row pgx.Row) (*dto.Chat, error) { + var chat dto.Chat + if err := row.Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs); err != nil { + return nil, apperr.WrapPostgres(err, "failed to get chat") + } + return &chat, nil +} diff --git a/service/internal/adapter/repository/storage/idempotency_test.go b/service/internal/adapter/repository/storage/idempotency_test.go new file mode 100644 index 0000000..db5164c --- /dev/null +++ b/service/internal/adapter/repository/storage/idempotency_test.go @@ -0,0 +1,303 @@ +package storage + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/migrator" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + chatsvc "gitea.privcloud.tech/mtusi2026/internal/service/chat" + "github.com/google/uuid" +) + +// The explicit runner supplies a newly created database; never alter the API database. +func TestIdempotencyTransactionAndExpiry(t *testing.T) { + database := os.Getenv("TEST_DATABASE_NAME") + if database == "" { + t.Skip("run make test-storage for an isolated PostgreSQL database") + } + if database == os.Getenv("POSTGRES_DATABASE") { + t.Fatal("test database must differ from application database") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cfg, err := config.New() + if err != nil { + t.Fatal(err) + } + cfg.Postgres.Database = database + cfg.Postgres.RunMigrations = true + if err = migrator.New(cfg).Start(ctx); err != nil { + t.Fatal(err) + } + pool := postgres.New(cfg.Postgres) + defer pool.Close() + repo := NewMessagesRepo(pool) + user, chat := uuid.New(), uuid.New() + if _, err = pool.Exec(ctx, `INSERT INTO users(id,name,username,password_hash) VALUES($1,'Synthetic','atomic_test','unused')`, user); err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `INSERT INTO chats(id,title) VALUES($1,'Atomic test')`, chat); err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)`, chat, user); err != nil { + t.Fatal(err) + } + req := dto.CreateMessage{ChatID: chat, SenderID: user, Text: "atomic", IdempotencyKey: "atomic-key"} + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil { + t.Fatal(err) + } + if err = tx.Rollback(ctx); err != nil { + t.Fatal(err) + } + var messages, keys int + if err = pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM messages),(SELECT count(*) FROM message_idempotency)`).Scan(&messages, &keys); err != nil { + t.Fatal(err) + } + if messages != 0 || keys != 0 { + t.Fatalf("rollback left %d messages, %d keys", messages, keys) + } + tx, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + first, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req) + if err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + // Expire the test record only: no shortened production TTL and no wall-clock wait. + if _, err = pool.Exec(ctx, `UPDATE message_idempotency SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=$1`, user); err != nil { + t.Fatal(err) + } + req.Text = "new after expiry" + tx, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + second, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req) + if err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if first.ID == second.ID { + t.Fatal("expired key replayed old message") + } + if err = pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM messages),(SELECT count(*) FROM message_idempotency)`).Scan(&messages, &keys); err != nil { + t.Fatal(err) + } + if messages != 2 || keys != 1 { + t.Fatalf("expiry left %d messages, %d keys; want 2,1", messages, keys) + } + if _, err = pool.Exec(ctx, `CREATE FUNCTION reject_test_key() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='failure-key' THEN RAISE EXCEPTION 'synthetic failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER reject_test_key BEFORE INSERT ON message_idempotency FOR EACH ROW EXECUTE FUNCTION reject_test_key()`); err != nil { + t.Fatal(err) + } + req.IdempotencyKey = "failure-key" + tx, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err == nil { + t.Fatal("expected injected failure") + } + if err = tx.Rollback(ctx); err != nil { + t.Fatal(err) + } + if err = pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM messages),(SELECT count(*) FROM message_idempotency)`).Scan(&messages, &keys); err != nil { + t.Fatal(err) + } + if messages != 2 || keys != 1 { + t.Fatal("failed result persisted a side effect") + } + if _, err = pool.Exec(ctx, `DROP TRIGGER reject_test_key ON message_idempotency; DROP FUNCTION reject_test_key()`); err != nil { + t.Fatal(err) + } + tx, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil { + t.Fatal("transient failure was cached", err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + + t.Run("service owns message content rules", func(t *testing.T) { + svc := chatsvc.New(txmanager.New(pool), NewChatsRepo(pool), repo, uncachedChats{}) + own, foreign, missing := uuid.New(), uuid.New(), uuid.New() + otherChat := uuid.New() + if _, err := pool.Exec(ctx, `INSERT INTO chats(id,title) VALUES($1,'Other chat');`, otherChat); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)`, otherChat, user); err != nil { + t.Fatal(err) + } + for _, fixture := range []struct { + id, chat uuid.UUID + missing bool + }{{own, chat, false}, {foreign, otherChat, false}, {missing, chat, true}} { + if _, err := pool.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,integrity_missing,created_at,updated_at) VALUES($1,$2,$3,'test.png',$4,'image/png',1,$5,$6,now(),now())`, fixture.id, fixture.chat, user, "originals/"+fixture.id.String(), strings.Repeat("a", 64), fixture.missing); err != nil { + t.Fatal(err) + } + } + counts := func() string { + t.Helper() + var messages, links, keys int + if err := pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM messages),(SELECT count(*) FROM message_attachments),(SELECT count(*) FROM message_idempotency)`).Scan(&messages, &links, &keys); err != nil { + t.Fatal(err) + } + return fmt.Sprintf("%d/%d/%d", messages, links, keys) + } + for _, tc := range []struct { + name, text string + ids []uuid.UUID + code apperr.Code + }{ + {"empty", "", nil, apperr.CodeBadRequest}, + {"whitespace", " \t\n\u2003", nil, apperr.CodeBadRequest}, + {"invalid UTF8", string([]byte{0xff}), nil, apperr.CodeBadRequest}, + {"nul", "\x00", nil, apperr.CodeBadRequest}, + {"too long", strings.Repeat("я", 4001), nil, apperr.CodeBadRequest}, + {"duplicate", "", []uuid.UUID{own, own}, apperr.CodeBadRequest}, + {"too many", "text", make([]uuid.UUID, 11), apperr.CodeBadRequest}, + {"nil id", "", []uuid.UUID{uuid.Nil}, apperr.CodeBadRequest}, + {"unknown", "", []uuid.UUID{uuid.New()}, apperr.CodeNotFound}, + {"foreign chat", "", []uuid.UUID{foreign}, apperr.CodeNotFound}, + {"missing object", "", []uuid.UUID{missing}, apperr.CodeUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + before := counts() + _, err := svc.CreateMessage(ctx, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: tc.text, AttachmentIDs: tc.ids}) + if err == nil || apperr.CodeFromError(err) != tc.code { + t.Fatalf("got %v, want code %v", err, tc.code) + } + if after := counts(); after != before { + t.Fatalf("rejected message changed counts: %s -> %s", before, after) + } + }) + } + for _, text := range []string{"", " \t\u2003"} { + request := dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: text, AttachmentIDs: []uuid.UUID{own}} + first, err := svc.CreateMessage(ctx, request) + if err != nil { + t.Fatal(err) + } + replay, err := svc.CreateMessage(ctx, request) + if err != nil || replay.ID != first.ID || len(replay.AttachmentIDs) != 1 || replay.AttachmentIDs[0] != own { + t.Fatalf("attachment replay failed: %v, %v", replay, err) + } + page, err := svc.ListForChat(ctx, dto.MessageFilters{UserID: user, ChatID: chat}) + if err != nil { + t.Fatal(err) + } + found := false + for _, msg := range page.Items { + if msg.ID == first.ID { + found = len(msg.AttachmentIDs) == 1 && msg.AttachmentIDs[0] == own + } + } + if !found { + t.Fatal("history lost attachment") + } + request.AttachmentIDs = []uuid.UUID{foreign} + if _, err = svc.CreateMessage(ctx, request); apperr.CodeFromError(err) != apperr.CodeNotFound { + t.Fatalf("foreign replay: %v", err) + } + } + // Preserve legacy hashes written before the attachment migration. + legacy := dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "legacy"} + first, err := svc.CreateMessage(ctx, legacy) + if err != nil { + t.Fatal(err) + } + hash := sha256.Sum256([]byte(legacy.Text)) + if _, err = pool.Exec(ctx, `UPDATE message_idempotency SET request_hash=$1,hash_version=1 WHERE key=$2`, hash[:], legacy.IdempotencyKey); err != nil { + t.Fatal(err) + } + replay, err := svc.CreateMessage(ctx, legacy) + if err != nil || replay.ID != first.ID { + t.Fatalf("legacy replay failed: %v", err) + } + legacy.AttachmentIDs = []uuid.UUID{own} + if _, err = svc.CreateMessage(ctx, legacy); apperr.CodeFromError(err) != apperr.CodeConflict { + t.Fatalf("legacy hash ignored new attachment: %v", err) + } + textOnly := dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "text only"} + first, err = svc.CreateMessage(ctx, textOnly) + if err != nil { + t.Fatal(err) + } + textOnly.AttachmentIDs = []uuid.UUID{} + replay, err = svc.CreateMessage(ctx, textOnly) + if err != nil || replay.ID != first.ID { + t.Fatalf("nil and empty attachment lists differ: %v", err) + } + // Force link failure using the relational FK and prove all preceding writes roll back. + before := counts() + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}}) + if err == nil { + t.Fatal("expected missing attachment FK failure") + } + _ = tx.Rollback(ctx) + if after := counts(); after != before { + t.Fatalf("failed link changed counts: %s -> %s", before, after) + } + var triggers int + if err = pool.QueryRow(ctx, `SELECT count(*) FROM pg_trigger WHERE tgname='message_content'`).Scan(&triggers); err != nil || triggers != 0 { + t.Fatalf("message trigger remains: %d, %v", triggers, err) + } + }) + + t.Run("object lock cancellation does not leak a pooled lock", func(t *testing.T) { + attachments := &AttachmentsRepo{Pool: pool} + key := "originals/" + uuid.NewString() + _, unlock, err := attachments.Lock(ctx, key) + if err != nil { + t.Fatal(err) + } + waiting, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + _, _, err = attachments.Lock(waiting, key) + cancel() + unlock() + if err == nil { + t.Fatal("contending lock ignored deadline") + } + _, release, err := attachments.Lock(ctx, key) + if err != nil { + t.Fatalf("lock leaked after cancellation: %v", err) + } + release() + }) + + t.Log("rollback removed both side effect and key; expired key created one new message") +} + +type uncachedChats struct{} + +func (uncachedChats) Create(context.Context, dto.Chat) error { return nil } +func (uncachedChats) Get(context.Context, uuid.UUID) (*dto.Chat, error) { + return nil, errors.New("cache disabled for storage test") +} diff --git a/service/internal/adapter/repository/storage/jobs.go b/service/internal/adapter/repository/storage/jobs.go new file mode 100644 index 0000000..887a59c --- /dev/null +++ b/service/internal/adapter/repository/storage/jobs.go @@ -0,0 +1,134 @@ +package storage + +import ( + "context" + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "time" +) + +func (r *AttachmentsRepo) Ping(ctx context.Context) error { return r.Pool.Ping(ctx) } +func (r *AttachmentsRepo) GetTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentRecord, error) { + return ScanAttachment(tx.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1 FOR UPDATE", id)) +} +func (r *AttachmentsRepo) InsertTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error { + p := a.Public + _, e := tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,staging_key,status) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$10,$11)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.StagingKey, p.Status) + return e +} +func (r *AttachmentsRepo) UpdateTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error { + variants, e := json.Marshal(a.Public.Variants) + if e != nil { + return e + } + keys, e := json.Marshal(a.VariantKeys) + if e != nil { + return e + } + _, e = tx.Exec(ctx, `UPDATE attachments SET status=$2,error_code=$3,pipeline_version=$4,staging_key=$5,object_key=$6,variants=$7,variant_keys=$8,updated_at=$9,integrity_missing=$10 WHERE id=$1`, a.Public.Id, a.Public.Status, a.Public.ErrorCode, a.PipelineVersion, a.StagingKey, a.Key, variants, keys, a.Public.UpdatedAt, a.Missing) + return e +} + +const jobColumns = `id,attachment_id,pipeline_version,attempt,created_at,state,token,lease_until,next_attempt_at,attempt_limit,error_code,updated_at,traceparent,tracestate` + +func scanJob(row pgx.Row) (*dto.AttachmentJob, error) { + j := new(dto.AttachmentJob) + m := &j.Message + err := row.Scan(&m.JobID, &m.AttachmentID, &m.PipelineVersion, &m.Attempt, &m.CreatedAt, &j.State, &j.Token, &j.LeaseUntil, &j.NextAttemptAt, &j.AttemptLimit, &j.ErrorCode, &j.UpdatedAt, &m.Traceparent, &m.Tracestate) + m.SchemaVersion = 1 + m.Type = "attachment.process.v1" + return j, err +} +func (r *AttachmentsRepo) JobTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentJob, error) { + return scanJob(tx.QueryRow(ctx, `SELECT `+jobColumns+` FROM attachment_jobs WHERE id=$1 FOR UPDATE`, id)) +} +func (r *AttachmentsRepo) InsertJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error { + m := j.Message + _, e := tx.Exec(ctx, `INSERT INTO attachment_jobs(`+jobColumns+`) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)`, m.JobID, m.AttachmentID, m.PipelineVersion, m.Attempt, m.CreatedAt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt, m.Traceparent, m.Tracestate) + return e +} +func (r *AttachmentsRepo) SaveJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error { + _, e := tx.Exec(ctx, `UPDATE attachment_jobs SET attempt=$2,state=$3,token=$4,lease_until=$5,next_attempt_at=$6,attempt_limit=$7,error_code=$8,updated_at=$9,traceparent=$10,tracestate=$11 WHERE id=$1`, j.Message.JobID, j.Message.Attempt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt, j.Message.Traceparent, j.Message.Tracestate) + return e +} +func (r *AttachmentsRepo) ExpiredJobs(ctx context.Context) ([]uuid.UUID, error) { + rows, e := r.Pool.Query(ctx, `SELECT id FROM attachment_jobs WHERE state='processing' AND lease_until>'attempt')::integer=j.attempt LIMIT 100`) + if e != nil { + return nil, e + } + defer rows.Close() + var ids []uuid.UUID + for rows.Next() { + var id uuid.UUID + if e = rows.Scan(&id); e != nil { + return nil, e + } + ids = append(ids, id) + } + return ids, rows.Err() +} +func (r *AttachmentsRepo) RetiredSources(ctx context.Context, before time.Time) ([]*dto.AttachmentRecord, error) { + rows, e := r.Pool.Query(ctx, `SELECT `+AttachmentColumns+` FROM attachments WHERE staging_key<>'' AND (status='ready' OR (status='failed' AND updated_at<$1)) LIMIT 100`, before) + if e != nil { + return nil, e + } + defer rows.Close() + var out []*dto.AttachmentRecord + for rows.Next() { + a, e := ScanAttachment(rows) + if e != nil { + return nil, e + } + out = append(out, a) + } + return out, rows.Err() +} + +func (r *AttachmentsRepo) SaveVariantTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, version int, name, key, sha string, size int64) error { + _, e := tx.Exec(ctx, `INSERT INTO attachment_variants(attachment_id,pipeline_version,variant,object_key,sha256,size_bytes) VALUES($1,$2,$3,$4,$5,$6)`, id, version, name, key, sha, size) + return e +} + +func (r *AttachmentsRepo) PublicationDeadTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, attempt int) (bool, error) { + var dead bool + e := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM outbox WHERE job_id=$1 AND status='dead_letter' AND (payload->>'attempt')::integer=$2)`, id, attempt).Scan(&dead) + return dead, e +} diff --git a/service/internal/adapter/repository/storage/messages.go b/service/internal/adapter/repository/storage/messages.go new file mode 100644 index 0000000..0c1b6f9 --- /dev/null +++ b/service/internal/adapter/repository/storage/messages.go @@ -0,0 +1,135 @@ +package storage + +import ( + "context" + + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" +) + +type MessagesRepo struct { + pool *pgxpool.Pool +} + +func NewMessagesRepo(pool *pgxpool.Pool) *MessagesRepo { + return &MessagesRepo{pool: pool} +} + +// LockKeyTx serializes persistence for one idempotency scope. +func (r *MessagesRepo) LockKeyTx(ctx context.Context, tx pgx.Tx, scope string) error { + _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, scope) + if err != nil { + return apperr.WrapPostgres(err, "failed to lock message key") + } + return nil +} +func (r *MessagesRepo) ReplayTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.MessageReplay, error) { + v := new(dto.MessageReplay) + err := tx.QueryRow(ctx, `SELECT request_hash,message_id,hash_version FROM message_idempotency WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`, req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&v.Hash, &v.MessageID, &v.HashVersion) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to read message key") + } + return v, nil +} +func (r *MessagesRepo) MessageTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.Message, error) { + return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, id)) +} +func (r *MessagesRepo) InsertMessageTx(ctx context.Context, tx pgx.Tx, message dto.Message) error { + _, err := tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`, message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt) + if err != nil { + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && pgErr.Code == pgerrcode.ForeignKeyViolation { + return apperr.Wrap(err, apperr.CodeNotFound, "chat not found") + } + return apperr.WrapPostgres(err, "failed to create message") + } + for position, id := range message.AttachmentIDs { + if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, id, position); err != nil { + return apperr.WrapPostgres(err, "failed to link attachment") + } + } + return nil +} +func (r *MessagesRepo) SaveKeyTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage, hash []byte, messageID uuid.UUID, expires time.Time) error { + _, err := tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version) VALUES($1,'POST',$2,$3,$4,$5,$6,2) ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash,message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash, messageID, expires) + if err != nil { + return apperr.WrapPostgres(err, "failed to save message key") + } + return nil +} + +func (r *MessagesRepo) GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) { + const query = `SELECT ` + messageColumns + ` FROM messages WHERE chat_id = $1 AND id = $2` + return scanMessage(r.pool.QueryRow(ctx, query, chatID, messageID)) +} + +func (r *MessagesRepo) ListForChat(ctx context.Context, chatID uuid.UUID, page dto.PageQuery) ([]dto.Message, error) { + query := `SELECT ` + messageColumns + ` FROM messages WHERE chat_id = $1` + args := []any{chatID, page.Limit} + if page.After != nil { + query += ` AND (created_at, id) > ($3, $4)` + args = append(args, page.After.CreatedAt, page.After.ID) + } + query += ` ORDER BY created_at, id LIMIT $2` + rows, err := r.pool.Query(ctx, query, args...) + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to list messages") + } + defer rows.Close() + messages := make([]dto.Message, 0) + for rows.Next() { + message, err := scanMessage(rows) + if err != nil { + return nil, err + } + messages = append(messages, *message) + } + if err = rows.Err(); err != nil { + return nil, apperr.WrapPostgres(err, "failed to list messages") + } + return messages, nil +} + +func scanMessage(row pgx.Row) (*dto.Message, error) { + var message dto.Message + if err := row.Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, &message.CreatedAt, &message.AttachmentIDs); err != nil { + return nil, apperr.WrapPostgres(err, "failed to get message") + } + return &message, nil +} + +const messageColumns = `id, chat_id, sender_id, text, created_at, + ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=messages.id ORDER BY position)` + +func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) { + // Prevent a concurrent integrity update from invalidating the checked rows + // until the message and its links have committed together. + rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing,status FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids) + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to read attachments") + } + defer rows.Close() + result := make([]dto.MessageAttachment, 0, len(ids)) + for rows.Next() { + var a dto.MessageAttachment + if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing, &a.Status); err != nil { + return nil, apperr.WrapPostgres(err, "failed to read attachment") + } + result = append(result, a) + } + if err := rows.Err(); err != nil { + return nil, apperr.WrapPostgres(err, "failed to read attachments") + } + return result, nil +} diff --git a/service/internal/adapter/repository/storage/observation.go b/service/internal/adapter/repository/storage/observation.go new file mode 100644 index 0000000..2fe1527 --- /dev/null +++ b/service/internal/adapter/repository/storage/observation.go @@ -0,0 +1,51 @@ +package storage + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/service/observation" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "time" +) + +func (r *AttachmentsRepo) AcceptSLO(ctx context.Context, tx pgx.Tx, id uuid.UUID, accepted, deadline time.Time) error { + _, e := tx.Exec(ctx, `INSERT INTO job_slo(job_id,accepted_at,deadline_at)VALUES($1,$2,$3)`, id, accepted, deadline) + return e +} +func (r *AttachmentsRepo) ReadySLO(ctx context.Context, tx pgx.Tx, id uuid.UUID, ready time.Time) error { + _, e := tx.Exec(ctx, `UPDATE job_slo SET ready_at=COALESCE(ready_at,$2) WHERE job_id=$1`, id, ready) + return e +} + +type ObservationRepo struct{ Pool *pgxpool.Pool } + +func (r *ObservationRepo) Pending(ctx context.Context) ([]observation.Sample, error) { + rows, e := r.Pool.Query(ctx, `SELECT job_id,accepted_at,deadline_at,ready_at,failed_at FROM job_slo WHERE outcome IS NULL ORDER BY deadline_at LIMIT 500`) + if e != nil { + return nil, e + } + defer rows.Close() + out := []observation.Sample{} + for rows.Next() { + var x observation.Sample + if e = rows.Scan(&x.ID, &x.Accepted, &x.Deadline, &x.Ready, &x.Failed); e != nil { + return nil, e + } + out = append(out, x) + } + return out, rows.Err() +} +func (r *ObservationRepo) Decide(ctx context.Context, id uuid.UUID, outcome string, at time.Time) error { + _, e := r.Pool.Exec(ctx, `UPDATE job_slo SET outcome=$2,evaluated_at=$3 WHERE job_id=$1 AND outcome IS NULL`, id, outcome, at) + return e +} +func (r *ObservationRepo) Snapshot(ctx context.Context) (x observation.Snapshot, e error) { + e = r.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')), (SELECT count(*) FROM outbox WHERE status IN ('pending','processing','failed')), COALESCE((SELECT EXTRACT(EPOCH FROM clock_timestamp()-min(created_at)) FROM attachment_jobs WHERE state IN ('queued','processing')),0), (SELECT count(*) FROM job_slo WHERE outcome='good'), (SELECT count(*) FROM job_slo WHERE outcome='bad'), (SELECT count(*) FROM pg_stat_activity WHERE wait_event_type='Lock')`).Scan(&x.Queued, &x.Outbox, &x.Oldest, &x.Good, &x.Bad, &x.Locked) + return +} + +func (r *AttachmentsRepo) FailSLO(ctx context.Context, tx pgx.Tx, id uuid.UUID, at time.Time) error { + _, e := tx.Exec(ctx, `UPDATE job_slo SET failed_at=COALESCE(failed_at,$2) WHERE job_id=$1`, id, at) + return e +} diff --git a/service/internal/adapter/repository/storage/outbox.go b/service/internal/adapter/repository/storage/outbox.go new file mode 100644 index 0000000..ffc3191 --- /dev/null +++ b/service/internal/adapter/repository/storage/outbox.go @@ -0,0 +1,52 @@ +package storage + +import ( + "context" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "time" +) + +type OutboxRepo struct{ Pool *pgxpool.Pool } + +func (r *OutboxRepo) CreateTx(ctx context.Context, tx pgx.Tx, q outbox.CreateMessageReq) (outbox.CreateMessageRes, error) { + _, e := tx.Exec(ctx, `INSERT INTO outbox(id,job_id,payload,status,max_attempts,token,next_attempt_at,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$7,$7)`, q.ID, q.AggregateID, q.Payload, q.Status, q.MaxAttempts, uuid.New(), q.CreatedAt) + return outbox.CreateMessageRes{ID: q.ID, CreatedAt: q.CreatedAt}, e +} +func scanOutbox(rows pgx.Rows) (out []outbox.AcquiredMessage, err error) { + defer rows.Close() + for rows.Next() { + var m outbox.AcquiredMessage + if err = rows.Scan(&m.ID, &m.AggregateID, &m.Payload, &m.Attempts, &m.MaxAttempts, &m.Token); err != nil { + return + } + out = append(out, m) + } + err = rows.Err() + return +} +func (r *OutboxRepo) AcquireManyForProcessing(ctx context.Context, q outbox.AcquireMessagesForProcessingReq) ([]outbox.AcquiredMessage, error) { + rows, e := r.Pool.Query(ctx, `WITH picked AS (SELECT id FROM outbox WHERE status IN ('pending','failed') AND next_attempt_at<=clock_timestamp() ORDER BY next_attempt_at,id LIMIT $1 FOR UPDATE SKIP LOCKED) + UPDATE outbox o SET status='processing',attempts=attempts+1,token=$2,locked_until=clock_timestamp()+$3::interval,updated_at=clock_timestamp() FROM picked p WHERE o.id=p.id RETURNING o.id,o.job_id,o.payload,o.attempts,o.max_attempts,o.token`, q.Limit, uuid.New(), q.LockDuration.String()) + if e != nil { + return nil, e + } + return scanOutbox(rows) +} +func (r *OutboxRepo) Expired(ctx context.Context) ([]outbox.AcquiredMessage, error) { + rows, e := r.Pool.Query(ctx, `SELECT id,job_id,payload,attempts,max_attempts,token FROM outbox WHERE status='processing' AND locked_until 10485760 || c.MaxPixels < 1 || c.MaxPixels > 40000000 || c.Timeout < time.Second || c.Timeout > 120*time.Second || c.Grace <= 2*c.Timeout { + return nil, fmt.Errorf("invalid attachment limits or grace period") + } + return c, nil +} + +func env(k, d string) string { + if v := os.Getenv(k); v != "" { + return v + } + return d +} diff --git a/service/internal/config/config.go b/service/internal/config/config.go new file mode 100644 index 0000000..e67e907 --- /dev/null +++ b/service/internal/config/config.go @@ -0,0 +1,110 @@ +package config + +import ( + "strings" + "time" + + "github.com/spf13/viper" + "go.uber.org/multierr" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment" +) + +var defaultConfig = &Config{ + Server: &Server{ + Host: "localhost", + Port: 8001, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 10 * time.Second, + ShutdownTimeout: 10 * time.Second, + Enabled: false, + }, + Postgres: &Postgres{ + Host: "localhost", + Port: 5432, + User: "postgres", + Password: "postgres", + Database: "postgres", + VersionTable: "versions", + RunMigrations: false, + MaxConnections: 10, + ConnectTimeout: 500 * time.Millisecond, + QueryTimeout: 2 * time.Second, + }, + Redis: &Redis{ + Host: "localhost", + Port: 6379, + Username: "", + Password: "", + DB: 0, + }, + JWT: &JWT{ + AccessTokenLifetime: 4 * time.Hour, + RefreshTokenLifetime: time.Hour, + PublicKeyPath: "certs/private.pem", + PrivateKeyPath: "certs/public.pem", + }, + Application: &Application{ + LogLevel: "info", + ShutdownTimeout: 10 * time.Second, + Environment: environment.ApplicationEnvironmentDev, + RequestRateLimit: 100, + }, +} + +type Config struct { + Server *Server + Postgres *Postgres + Redis *Redis + JWT *JWT + Application *Application +} + +func New() (*Config, error) { + v := viper.New() + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + v.AutomaticEnv() + + cfg := defaultConfig.copy() + + if err := cfg.loadConfig(v); err != nil { + return nil, err + } + + if err := cfg.Validate(); err != nil { + return nil, err + } + + return cfg, nil +} + +func (c *Config) Validate() error { + return multierr.Combine( + c.Server.Validate(), + c.Postgres.Validate(), + c.JWT.Validate(), + c.Redis.Validate(), + c.Application.Validate(), + ) +} + +func (c *Config) loadConfig(v *viper.Viper) error { + return multierr.Combine( + c.Postgres.loadConfig(v), + c.Server.loadConfig(v), + c.JWT.loadConfig(v), + c.Redis.loadConfig(v), + c.Application.loadConfig(v), + ) +} + +func (c *Config) copy() *Config { + return &Config{ + Server: c.Server.copy(), + Postgres: c.Postgres.copy(), + Redis: c.Redis.copy(), + JWT: c.JWT.copy(), + Application: c.Application.copy(), + } +} diff --git a/service/internal/config/cookies.go b/service/internal/config/cookies.go new file mode 100644 index 0000000..b6f7d88 --- /dev/null +++ b/service/internal/config/cookies.go @@ -0,0 +1,65 @@ +package config + +import ( + "errors" + "net/http" + "strings" + + "github.com/spf13/viper" +) + +type Cookies struct { + Enabled bool + Domain string + Path string + Secure bool + SameSite http.SameSite + AllowedOrigins []string +} + +func (c *Cookies) Validate() error { + switch { + case c.Domain == "": + return wrapValidationError("cookies", errors.New("domain cannot be empty")) + case c.Path == "": + return wrapValidationError("cookies", errors.New("path cannot be empty")) + case c.SameSite == http.SameSiteDefaultMode: + return wrapValidationError("cookies", errors.New("same_site_set_to_none_mode cannot be empty")) + //case c.Enabled && !c.Secure: + // return wrapValidationError("cookies", errors.New("if cookies_enabled equals true then cookies_secure must be true")) + case c.AllowedOrigins == nil || len(c.AllowedOrigins) == 0: + return wrapValidationError("cookies", errors.New("allowed_origins cannot be empty")) + default: + return nil + } +} + +func (c *Cookies) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("COOKIES") + c.Enabled = v.GetBool("ENABLED") + c.Domain = v.GetString("DOMAIN") + c.Path = v.GetString("PATH") + c.Secure = v.GetBool("SECURE") + c.AllowedOrigins = v.GetStringSlice("ALLOWED_ORIGINS") + sameSite := v.GetString("SAME_SITE") + switch strings.ToLower(strings.TrimSpace(sameSite)) { + case "strict": + c.SameSite = http.SameSiteStrictMode + case "none": + c.SameSite = http.SameSiteNoneMode + default: + c.SameSite = http.SameSiteLaxMode + } + return nil +} + +func (c *Cookies) copy() *Cookies { + return &Cookies{ + Enabled: c.Enabled, + Domain: c.Domain, + Path: c.Path, + Secure: c.Secure, + SameSite: c.SameSite, + AllowedOrigins: c.AllowedOrigins, + } +} diff --git a/service/internal/config/jwt.go b/service/internal/config/jwt.go new file mode 100644 index 0000000..a48bdd6 --- /dev/null +++ b/service/internal/config/jwt.go @@ -0,0 +1,63 @@ +package config + +import ( + "errors" + "os" + "strconv" + "time" + + "github.com/spf13/viper" +) + +type JWT struct { + AccessTokenLifetime time.Duration + RefreshTokenLifetime time.Duration + PublicKeyPath string + PrivateKeyPath string +} + +func (c *JWT) Validate() error { + switch { + case c.AccessTokenLifetime <= 0: + return wrapValidationError("jwt", errors.New("access_token_lifetime cannot be empty")) + case c.RefreshTokenLifetime <= 0: + return wrapValidationError("jwt", errors.New("refresh_token_lifetime cannot be empty")) + case c.PrivateKeyPath == "": + return wrapValidationError("jwt", errors.New("private_key_path cannot be empty")) + case c.PublicKeyPath == "": + return wrapValidationError("jwt", errors.New("public_key_path cannot be empty")) + default: + return nil + } +} + +func (c *JWT) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("JWT") + c.AccessTokenLifetime = v.GetDuration("ACCESS_TOKEN_LIFETIME") + c.RefreshTokenLifetime = time.Hour + if value := os.Getenv("SESSION_TTL_SECONDS"); value != "" { + seconds, err := strconv.ParseInt(value, 10, 64) + if err != nil || seconds <= 0 || seconds > int64((1<<63-1)/time.Second) { + return wrapValidationError("session", errors.New("SESSION_TTL_SECONDS must be a positive integer duration in seconds")) + } + c.RefreshTokenLifetime = time.Duration(seconds) * time.Second + } else if legacy := v.GetString("REFRESH_TOKEN_LIFETIME"); legacy != "" { + duration, err := time.ParseDuration(legacy) + if err != nil { + return wrapValidationError("jwt", errors.New("invalid refresh_token_lifetime")) + } + c.RefreshTokenLifetime = duration + } + c.PublicKeyPath = v.GetString("PUBLIC_KEY_PATH") + c.PrivateKeyPath = v.GetString("PRIVATE_KEY_PATH") + return nil +} + +func (c *JWT) copy() *JWT { + return &JWT{ + AccessTokenLifetime: c.AccessTokenLifetime, + RefreshTokenLifetime: c.RefreshTokenLifetime, + PublicKeyPath: c.PublicKeyPath, + PrivateKeyPath: c.PrivateKeyPath, + } +} diff --git a/service/internal/config/pipeline.go b/service/internal/config/pipeline.go new file mode 100644 index 0000000..44a2cea --- /dev/null +++ b/service/internal/config/pipeline.go @@ -0,0 +1,48 @@ +package config + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +type Pipeline struct { + Concurrency, Prefetch, MaxAttempts int + Timeout, Lease, Retention time.Duration + BrokerURL string +} + +func LoadPipeline() (*Pipeline, error) { + c := &Pipeline{Concurrency: 2, Prefetch: 2, MaxAttempts: 5, Timeout: 60 * time.Second, Lease: 90 * time.Second, Retention: 24 * time.Hour} + for k, p := range map[string]*int{"WORKER_CONCURRENCY": &c.Concurrency, "WORKER_PREFETCH": &c.Prefetch, "JOB_MAX_ATTEMPTS": &c.MaxAttempts} { + if v := os.Getenv(k); v != "" { + n, e := strconv.Atoi(v) + if e != nil || n < 1 || n > 64 { + return nil, fmt.Errorf("invalid %s", k) + } + *p = n + } + } + for k, p := range map[string]*time.Duration{"JOB_TIMEOUT": &c.Timeout, "JOB_LEASE": &c.Lease, "STAGING_RETENTION": &c.Retention} { + if v := os.Getenv(k); v != "" { + d, e := time.ParseDuration(v) + if e != nil || d <= 0 { + return nil, fmt.Errorf("invalid %s", k) + } + *p = d + } + } + if c.Lease <= c.Timeout || c.Prefetch > c.Concurrency { + return nil, fmt.Errorf("lease must exceed timeout; prefetch must not exceed concurrency") + } + if p := os.Getenv("BROKER_URL_FILE"); p != "" { + b, e := os.ReadFile(p) + if e != nil { + return nil, fmt.Errorf("read broker secret: %w", e) + } + c.BrokerURL = strings.TrimSpace(string(b)) + } + return c, nil +} diff --git a/service/internal/config/postgres.go b/service/internal/config/postgres.go new file mode 100644 index 0000000..478642a --- /dev/null +++ b/service/internal/config/postgres.go @@ -0,0 +1,110 @@ +package config + +import ( + "errors" + "fmt" + "math" + "time" + + "github.com/spf13/viper" +) + +const ( + uri = "postgres://%s:%s@%s:%d/%s?sslmode=disable" +) + +type Postgres struct { + Host string + Port int + User string + Password string + Database string + VersionTable string + RunMigrations bool + MaxConnections int + ConnectTimeout time.Duration + QueryTimeout time.Duration +} + +func (c *Postgres) Validate() error { + switch { + case c.Host == "": + return wrapValidationError("postgres", errors.New("host cannot be empty")) + case c.Port == 0: + return wrapValidationError("postgres", errors.New("port cannot be empty")) + case c.User == "": + return wrapValidationError("postgres", errors.New("user cannot be empty")) + case c.Password == "": + return wrapValidationError("postgres", errors.New("password cannot be empty")) + case c.Database == "": + return wrapValidationError("postgres", errors.New("database cannot be empty")) + case c.VersionTable == "": + return wrapValidationError("postgres", errors.New("version_table name cannot be empty")) + case c.MaxConnections <= 0 || int64(c.MaxConnections) > math.MaxInt32: + return wrapValidationError("postgres", errors.New("max_connections must be between 1 and 2147483647")) + case c.ConnectTimeout <= 0: + return wrapValidationError("postgres", errors.New("connect_timeout must be positive")) + case c.QueryTimeout < time.Millisecond: + return wrapValidationError("postgres", errors.New("query_timeout must be at least 1ms")) + default: + return nil + } +} + +func (c *Postgres) GetMaxConnections() int32 { return int32(c.MaxConnections) } +func (c *Postgres) GetConnectTimeout() time.Duration { return c.ConnectTimeout } +func (c *Postgres) GetQueryTimeout() time.Duration { return c.QueryTimeout } + +func (c *Postgres) GetVersionTableName() string { + if c == nil { + return "" + } + return c.VersionTable +} + +func (c *Postgres) GetURI() string { + if c == nil { + return "" + } + return fmt.Sprintf(uri, c.User, c.Password, c.Host, c.Port, c.Database) +} + +func (c *Postgres) GetRunMigrations() bool { + if c == nil { + return false + } + return c.RunMigrations +} + +func (c *Postgres) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("POSTGRES") + c.Host = v.GetString("HOST") + c.Port = v.GetInt("PORT") + c.User = v.GetString("USER") + c.Password = v.GetString("PASSWORD") + c.Database = v.GetString("DATABASE") + c.VersionTable = v.GetString("VERSION_TABLE") + c.RunMigrations = v.GetBool("RUN_MIGRATIONS") + c.MaxConnections = v.GetInt("MAX_CONNECTIONS") + c.ConnectTimeout = v.GetDuration("CONNECT_TIMEOUT") + c.QueryTimeout = v.GetDuration("QUERY_TIMEOUT") + return nil +} + +func (c *Postgres) copy() *Postgres { + if c == nil { + return nil + } + return &Postgres{ + Host: c.Host, + Port: c.Port, + User: c.User, + Password: c.Password, + Database: c.Database, + MaxConnections: c.MaxConnections, + ConnectTimeout: c.ConnectTimeout, + QueryTimeout: c.QueryTimeout, + RunMigrations: c.RunMigrations, + VersionTable: c.VersionTable, + } +} diff --git a/service/internal/config/redis.go b/service/internal/config/redis.go new file mode 100644 index 0000000..cdeafa5 --- /dev/null +++ b/service/internal/config/redis.go @@ -0,0 +1,72 @@ +package config + +import ( + "errors" + "fmt" + "net" + "time" + + "github.com/redis/go-redis/v9" + "github.com/spf13/viper" +) + +type Redis struct { + Host string + Port int + Username string + Password string + DB int +} + +func (c *Redis) Validate() error { + switch { + case c.Host == "": + return wrapValidationError("redis", errors.New("host cannot be empty")) + case c.Port == 0: + return wrapValidationError("redis", errors.New("port cannot be empty")) + default: + return nil + } +} + +func (c *Redis) BuildRedisOptions() *redis.Options { + if c == nil { + return nil + } + return &redis.Options{ + Addr: net.JoinHostPort(c.Host, fmt.Sprintf("%d", c.Port)), + Username: c.Username, + Password: c.Password, + DB: c.DB, + DialTimeout: 200 * time.Millisecond, + ReadTimeout: 200 * time.Millisecond, + WriteTimeout: 200 * time.Millisecond, + PoolTimeout: 200 * time.Millisecond, + MaxRetries: -1, + DialerRetries: 1, + ContextTimeoutEnabled: true, + } +} + +func (c *Redis) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("REDIS") + c.Host = v.GetString("HOST") + c.Port = v.GetInt("PORT") + c.DB = v.GetInt("DB") + c.Password = v.GetString("PASSWORD") + c.Username = v.GetString("USERNAME") + return nil +} + +func (c *Redis) copy() *Redis { + if c == nil { + return nil + } + return &Redis{ + Host: c.Host, + Port: c.Port, + Username: c.Username, + Password: c.Password, + DB: c.DB, + } +} diff --git a/service/internal/config/server.go b/service/internal/config/server.go new file mode 100644 index 0000000..d3f9fbe --- /dev/null +++ b/service/internal/config/server.go @@ -0,0 +1,69 @@ +package config + +import ( + "errors" + "fmt" + "time" + + "github.com/spf13/viper" +) + +type Server struct { + Host string + Port int + ReadTimeout time.Duration + WriteTimeout time.Duration + IdleTimeout time.Duration + ShutdownTimeout time.Duration + Enabled bool +} + +func (c *Server) Validate() error { + if !c.Enabled { + return nil + } + switch { + case c.Host == "": + return wrapValidationError("server", errors.New("host cannot be empty")) + case c.Port == 0: + return wrapValidationError("server", errors.New("port cannot be empty")) + case c.ReadTimeout == 0: + return wrapValidationError("server", errors.New("read_timeout cannot be empty")) + case c.WriteTimeout == 0: + return wrapValidationError("server", errors.New("write_timeout cannot be empty")) + case c.IdleTimeout == 0: + return wrapValidationError("server", errors.New("idle_timeout cannot be empty")) + case c.ShutdownTimeout == 0: + return wrapValidationError("server", errors.New("shutdown_timeout name cannot be empty")) + default: + return nil + } +} + +func (c *Server) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("APP") + c.Host = v.GetString("HOST") + c.Port = v.GetInt("PORT") + c.ReadTimeout = v.GetDuration("READ_TIMEOUT") + c.WriteTimeout = v.GetDuration("WRITE_TIMEOUT") + c.IdleTimeout = v.GetDuration("IDLE_TIMEOUT") + c.ShutdownTimeout = v.GetDuration("SHUTDOWN_TIMEOUT") + c.Enabled = v.GetBool("ENABLED") + return c.Validate() +} + +func (c *Server) BindAddress() string { + return fmt.Sprintf("%s:%d", c.Host, c.Port) +} + +func (c *Server) copy() *Server { + return &Server{ + Host: c.Host, + Port: c.Port, + ReadTimeout: c.ReadTimeout, + WriteTimeout: c.WriteTimeout, + IdleTimeout: c.IdleTimeout, + ShutdownTimeout: c.ShutdownTimeout, + Enabled: c.Enabled, + } +} diff --git a/service/internal/config/validation.go b/service/internal/config/validation.go new file mode 100644 index 0000000..aeb4502 --- /dev/null +++ b/service/internal/config/validation.go @@ -0,0 +1,11 @@ +package config + +import "fmt" + +func wrapValidationError(name string, err error) error { + if err == nil { + return nil + } + + return fmt.Errorf("%s validation error: %w", name, err) +} diff --git a/service/internal/dto/attachment.go b/service/internal/dto/attachment.go new file mode 100644 index 0000000..1469e14 --- /dev/null +++ b/service/internal/dto/attachment.go @@ -0,0 +1,48 @@ +package dto + +import ( + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "github.com/google/uuid" + "io" + "time" +) + +type AttachmentRecord struct { + Public contracts.Attachment + Key string + StagingKey string + PipelineVersion int + VariantKeys map[string]string + Missing bool +} +type UploadAttachment struct { + UserID, ChatID uuid.UUID + Filename, ContentType string + Body io.Reader +} +type AttachmentContent struct { + Body io.ReadCloser + ContentType string + Size int64 +} +type JobMessage struct { + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` + SchemaVersion int `json:"schema_version"` + JobID uuid.UUID `json:"job_id"` + Type string `json:"type"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` +} +type AttachmentJob struct { + Message JobMessage + State string + Token uuid.UUID + LeaseUntil *time.Time + NextAttemptAt time.Time + AttemptLimit int + ErrorCode *string + UpdatedAt time.Time +} diff --git a/service/internal/dto/auth.go b/service/internal/dto/auth.go new file mode 100644 index 0000000..5b56686 --- /dev/null +++ b/service/internal/dto/auth.go @@ -0,0 +1,126 @@ +package dto + +import ( + "errors" + "net/netip" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +// ---- Tokens ---- + +type UserDataInToken struct { + TokenID uuid.UUID + UserID uuid.UUID + Role role.Role + IsAccess bool + + PhoneNumber *string + Email *string + + ExpiresAt time.Time + IssuedAt time.Time +} + +func (u UserDataInToken) Validate() error { + if u.TokenID == uuid.Nil { + return errors.New("invalid token id") + } + if u.UserID == uuid.Nil { + return errors.New("invalid user id") + } + if !u.Role.IsValid() { + return errors.New("invalid role") + } + if u.ExpiresAt.IsZero() { + return errors.New("invalid expiry") + } + if u.IssuedAt.IsZero() { + return errors.New("invalid issued") + } + return nil +} + +type Token struct { + Payload string +} + +type ( + AccessToken Token + RefreshToken Token +) + +type Tokens struct { + AccessToken AccessToken + RefreshToken RefreshToken +} + +func (t AccessToken) Token() *Token { + return new(Token(t)) +} + +func (t RefreshToken) Token() *Token { + return new(Token(t)) +} + +type StoredRefreshToken struct { + ID uuid.UUID + UserID uuid.UUID + TokenHash string + UserAgent string + IPAddress netip.Addr + Role role.Role + ExpiresAt time.Time + RevokedAt *time.Time + CreatedAt time.Time +} + +type RevokeRefreshToken struct { + ID uuid.UUID + RevokedAt time.Time +} + +type RevokeRefreshTokensByUserID struct { + UserID uuid.UUID + RevokedAt time.Time +} + +// ---- Auth ---- + +type Session struct { + ExpiresAt time.Time + Token string + TokenType string + User User +} + +type Login struct { + Username string + Password string +} + +type LoginResult struct { + ExpiresAt time.Time + TokenType string + User User + Tokens Tokens +} + +// UserCredentials is internal authentication data, never a public user response. +type UserCredentials struct { + User User + PasswordHash string `json:"-"` +} + +type Logout struct { + Token string +} + +type Register struct { + Email string + Password string + RepeatPassword string +} diff --git a/service/internal/dto/chat.go b/service/internal/dto/chat.go new file mode 100644 index 0000000..eaf6bc8 --- /dev/null +++ b/service/internal/dto/chat.go @@ -0,0 +1,31 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +type Chat struct { + ID uuid.UUID + Title string + MemberIDs []uuid.UUID + CreatedAt time.Time +} + +type CreateChat struct { + CreatorID uuid.UUID + Title string + MemberIDs []uuid.UUID +} + +type ChatMember struct { + UserID uuid.UUID + ChatID uuid.UUID +} + +type ChatFilters struct { + UserID uuid.UUID + Limit *int + AfterID *uuid.UUID +} diff --git a/service/internal/dto/message.go b/service/internal/dto/message.go new file mode 100644 index 0000000..48f33b6 --- /dev/null +++ b/service/internal/dto/message.go @@ -0,0 +1,45 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +type Message struct { + AttachmentIDs []uuid.UUID + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string + CreatedAt time.Time +} + +type CreateMessage struct { + AttachmentIDs []uuid.UUID + IdempotencyKey string + ChatID uuid.UUID + SenderID uuid.UUID + Text string +} + +type MessageFilters struct { + ChatID uuid.UUID + UserID uuid.UUID + Limit *int + AfterID *uuid.UUID +} + +// MessageAttachment contains only the data needed to authorize linking an attachment. +type MessageAttachment struct { + ID uuid.UUID + ChatID uuid.UUID + Missing bool + Status string +} + +type MessageReplay struct { + Hash []byte + HashVersion int + MessageID uuid.UUID +} diff --git a/service/internal/dto/page.go b/service/internal/dto/page.go new file mode 100644 index 0000000..2af3f60 --- /dev/null +++ b/service/internal/dto/page.go @@ -0,0 +1,25 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +// Page is a page of records with the ID to use for the next page. +type Page[T any] struct { + Items []T + NextCursorID *uuid.UUID +} + +// PagePosition is a resolved database cursor, ordered by creation time and ID. +type PagePosition struct { + CreatedAt time.Time + ID uuid.UUID +} + +// PageQuery is prepared by a service; Limit includes the lookahead row. +type PageQuery struct { + Limit int + After *PagePosition +} diff --git a/service/internal/dto/user.go b/service/internal/dto/user.go new file mode 100644 index 0000000..b617e0a --- /dev/null +++ b/service/internal/dto/user.go @@ -0,0 +1,30 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +type User struct { + ID uuid.UUID + Username string + DisplayName string + CreatedAt time.Time +} + +type CreateUser struct { + Username string + DisplayName string + Password string +} + +// CreateStoredUser contains the prepared database record, never a plaintext password. +type CreateStoredUser struct { + ID uuid.UUID + Username string + DisplayName string + PasswordHash string `json:"-"` + CreatedAt time.Time + UpdatedAt time.Time +} diff --git a/service/internal/gen/contracts/types.gen.go b/service/internal/gen/contracts/types.gen.go new file mode 100644 index 0000000..977fa8c --- /dev/null +++ b/service/internal/gen/contracts/types.gen.go @@ -0,0 +1,232 @@ +// Package contracts provides primitives to interact with the openapi HTTP API. +// +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.5.1 DO NOT EDIT. +package contracts + +import ( + "time" + + openapi_types "github.com/oapi-codegen/runtime/types" +) + +const ( + BasicAuthScopes = "basicAuth.Scopes" + BearerAuthScopes = "bearerAuth.Scopes" +) + +// Defines values for AttachmentContentType. +const ( + AttachmentContentTypeImagejpeg AttachmentContentType = "image/jpeg" + AttachmentContentTypeImagepng AttachmentContentType = "image/png" +) + +// Defines values for AttachmentStatus. +const ( + Failed AttachmentStatus = "failed" + Processing AttachmentStatus = "processing" + Queued AttachmentStatus = "queued" + Ready AttachmentStatus = "ready" +) + +// Defines values for ErrorErrorCode. +const ( + Conflict ErrorErrorCode = "conflict" + InternalError ErrorErrorCode = "internal_error" + InvalidRequest ErrorErrorCode = "invalid_request" + NotFound ErrorErrorCode = "not_found" + PayloadTooLarge ErrorErrorCode = "payload_too_large" + RateLimited ErrorErrorCode = "rate_limited" + Unauthorized ErrorErrorCode = "unauthorized" + Unavailable ErrorErrorCode = "unavailable" + UnsupportedMediaType ErrorErrorCode = "unsupported_media_type" +) + +// Defines values for HealthStatus. +const ( + Ok HealthStatus = "ok" +) + +// Defines values for VariantContentType. +const ( + VariantContentTypeImagejpeg VariantContentType = "image/jpeg" + VariantContentTypeImagepng VariantContentType = "image/png" + VariantContentTypeImagewebp VariantContentType = "image/webp" +) + +// Defines values for VariantName. +const ( + VariantNameCrop VariantName = "crop" + VariantNameOptimized VariantName = "optimized" + VariantNameThumbnail VariantName = "thumbnail" +) + +// Defines values for DownloadAttachmentParamsVariant. +const ( + DownloadAttachmentParamsVariantCrop DownloadAttachmentParamsVariant = "crop" + DownloadAttachmentParamsVariantOptimized DownloadAttachmentParamsVariant = "optimized" + DownloadAttachmentParamsVariantOriginal DownloadAttachmentParamsVariant = "original" + DownloadAttachmentParamsVariantThumbnail DownloadAttachmentParamsVariant = "thumbnail" +) + +// Attachment defines model for Attachment. +type Attachment struct { + ChatId openapi_types.UUID `json:"chat_id"` + ContentType AttachmentContentType `json:"content_type"` + CreatedAt time.Time `json:"created_at"` + ErrorCode *string `json:"error_code"` + Filename string `json:"filename"` + Id openapi_types.UUID `json:"id"` + OwnerId openapi_types.UUID `json:"owner_id"` + Sha256 string `json:"sha256"` + SizeBytes int `json:"size_bytes"` + Status AttachmentStatus `json:"status"` + UpdatedAt time.Time `json:"updated_at"` + Variants []Variant `json:"variants"` +} + +// AttachmentContentType defines model for Attachment.ContentType. +type AttachmentContentType string + +// AttachmentStatus defines model for Attachment.Status. +type AttachmentStatus string + +// Chat defines model for Chat. +type Chat struct { + CreatedAt time.Time `json:"created_at"` + Id openapi_types.UUID `json:"id"` + MemberIds []openapi_types.UUID `json:"member_ids"` + Title string `json:"title"` +} + +// ChatPage defines model for ChatPage. +type ChatPage struct { + Items []Chat `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +// CreateChat defines model for CreateChat. +type CreateChat struct { + MemberIds []openapi_types.UUID `json:"member_ids"` + Title string `json:"title"` +} + +// CreateMessage defines model for CreateMessage. +type CreateMessage struct { + AttachmentIds *[]openapi_types.UUID `json:"attachment_ids,omitempty"` + Text *string `json:"text,omitempty"` +} + +// CreateUser defines model for CreateUser. +type CreateUser struct { + DisplayName string `json:"display_name"` + Password *string `json:"password,omitempty"` + Username string `json:"username"` +} + +// Error defines model for Error. +type Error struct { + Error struct { + Code ErrorErrorCode `json:"code"` + Message string `json:"message"` + RequestId string `json:"request_id"` + } `json:"error"` +} + +// ErrorErrorCode defines model for Error.Error.Code. +type ErrorErrorCode string + +// Health defines model for Health. +type Health struct { + Status HealthStatus `json:"status"` +} + +// HealthStatus defines model for Health.Status. +type HealthStatus string + +// Message defines model for Message. +type Message struct { + AttachmentIds []openapi_types.UUID `json:"attachment_ids"` + ChatId openapi_types.UUID `json:"chat_id"` + CreatedAt time.Time `json:"created_at"` + Id openapi_types.UUID `json:"id"` + SenderId openapi_types.UUID `json:"sender_id"` + Text string `json:"text"` +} + +// MessagePage defines model for MessagePage. +type MessagePage struct { + Items []Message `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +// Session defines model for Session. +type Session struct { + ExpiresAt time.Time `json:"expires_at"` + Token string `json:"token"` + TokenType string `json:"token_type"` + User User `json:"user"` +} + +// User defines model for User. +type User struct { + CreatedAt time.Time `json:"created_at"` + DisplayName string `json:"display_name"` + Id openapi_types.UUID `json:"id"` + Username string `json:"username"` +} + +// Variant defines model for Variant. +type Variant struct { + ContentType VariantContentType `json:"content_type"` + Height int `json:"height"` + Name VariantName `json:"name"` + Sha256 string `json:"sha256"` + SizeBytes int `json:"size_bytes"` + Width int `json:"width"` +} + +// VariantContentType defines model for Variant.ContentType. +type VariantContentType string + +// VariantName defines model for Variant.Name. +type VariantName string + +// DownloadAttachmentParams defines parameters for DownloadAttachment. +type DownloadAttachmentParams struct { + Variant *DownloadAttachmentParamsVariant `form:"variant,omitempty" json:"variant,omitempty"` +} + +// DownloadAttachmentParamsVariant defines parameters for DownloadAttachment. +type DownloadAttachmentParamsVariant string + +// ListChatsParams defines parameters for ListChats. +type ListChatsParams struct { + Limit *int `form:"limit,omitempty" json:"limit,omitempty"` + Cursor *string `form:"cursor,omitempty" json:"cursor,omitempty"` +} + +// UploadAttachmentParams defines parameters for UploadAttachment. +type UploadAttachmentParams struct { + Filename string `form:"filename" json:"filename"` +} + +// ListMessagesParams defines parameters for ListMessages. +type ListMessagesParams struct { + Limit *int `form:"limit,omitempty" json:"limit,omitempty"` + Cursor *string `form:"cursor,omitempty" json:"cursor,omitempty"` +} + +// CreateMessageParams defines parameters for CreateMessage. +type CreateMessageParams struct { + // IdempotencyKey Обязателен с lab3. Область: пользователь + метод + путь + ключ; хранить результат не менее 24 ч. Повтор того же JSON возвращает исходный 201 и то же сообщение, другой JSON — 409. + IdempotencyKey string `json:"Idempotency-Key"` +} + +// CreateChatJSONRequestBody defines body for CreateChat for application/json ContentType. +type CreateChatJSONRequestBody = CreateChat + +// CreateMessageJSONRequestBody defines body for CreateMessage for application/json ContentType. +type CreateMessageJSONRequestBody = CreateMessage + +// RegisterUserJSONRequestBody defines body for RegisterUser for application/json ContentType. +type RegisterUserJSONRequestBody = CreateUser diff --git a/service/internal/pkg/apperr/code.go b/service/internal/pkg/apperr/code.go new file mode 100644 index 0000000..429d640 --- /dev/null +++ b/service/internal/pkg/apperr/code.go @@ -0,0 +1,274 @@ +package apperr + +import ( + "errors" + "net/http" + + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + grpcCodes "google.golang.org/grpc/codes" +) + +//go:generate stringer -type=Code -linecomment +type Code uint16 + +const ( + // CodeBadRequest = {AppError: BAD_REQUEST, HTTP:400} + CodeBadRequest Code = iota + // CodePayloadTooLarge = {AppError: REQUEST_ENTITY_TOO_LARGE, HTTP:413} + CodePayloadTooLarge + // CodeValidation = {AppError: VALIDATION, HTTP:400} + CodeValidation + // CodeUnauthorized = {AppError: UNAUTHORIZED, HTTP:401} + CodeUnauthorized + // CodeForbidden = {AppError: FORBIDDEN, HTTP:403} + CodeForbidden + // CodeNotFound = {AppError: NOT_FOUND, HTTP:404} + CodeNotFound + // CodeAlreadyExists = {AppError: ALREADY_EXISTS, HTTP:409} + CodeAlreadyExists + // CodeConflict = {AppError: CONFLICT, HTTP:409} + CodeConflict + // CodeTooManyRequests = {AppError: TOO_MANY_REQUESTS, HTTP:429} + CodeTooManyRequests + // CodeTimeout = {AppError: TIMEOUT, HTTP:504} + CodeTimeout + // CodeInternal = {AppError: INTERNAL, HTTP:500} + CodeInternal + // CodeServiceUnavailable = {AppError: SERVICE_UNAVAILABLE, HTTP:503} + CodeServiceUnavailable + // CodeOK = {AppError: OK, HTTP:200} + CodeOK + // CodeLocked = {AppError: LOCKED, HTTP:423} + CodeLocked + // CodeUnsupportedMediaType = {AppError: UNSUPPORTED_MEDIA_TYPE, HTTP:415} + CodeUnsupportedMediaType + // CodeUnavailable = {AppError: SERVICE_UNAVAILABLE, HTTP: 503} + CodeUnavailable +) + +func (c Code) HTTPStatus() int { + if status, ok := httpStatusByCode[c]; ok { + return status + } + return http.StatusInternalServerError +} + +func (c Code) GRPCCode() grpcCodes.Code { + if code, ok := grpcCodeByCode[c]; ok { + return code + } + return grpcCodes.Internal +} + +func (c Code) PostgresCodes() []string { + codes := postgresCodesByCode[c] + if len(codes) == 0 { + return nil + } + + cp := make([]string, len(codes)) + copy(cp, codes) + return cp +} + +func (c Code) PostgresCode() string { + codes := c.PostgresCodes() + if len(codes) == 0 { + return "" + } + return codes[0] +} + +func CodeFromError(err error) Code { + if err == nil { + return CodeInternal + } + + var coded interface { + Code() Code + } + if errors.As(err, &coded) { + return coded.Code() + } + + if code, ok := codeFromRedisError(err, false, false); ok { + return code + } + + return CodeFromPostgresError(err) +} + +func HTTPStatusFromError(err error) int { + return CodeFromError(err).HTTPStatus() +} + +func GRPCCodeFromError(err error) grpcCodes.Code { + return CodeFromError(err).GRPCCode() +} + +func CodeFromHTTPStatus(status int) Code { + if code, ok := codeByHTTPStatus[status]; ok { + return code + } + return CodeInternal +} + +func CodeFromGRPC(code grpcCodes.Code) Code { + if appCode, ok := codeByGRPCCode[code]; ok { + return appCode + } + return CodeInternal +} + +func CodeFromPostgres(pgCode string) Code { + if code, ok := codeByPostgresCode[pgCode]; ok { + return code + } + return CodeInternal +} + +func CodeFromPostgresError(err error) Code { + if err == nil { + return CodeInternal + } + + if code, ok := codeByPgxError[err]; ok { + return code + } + + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok { + return CodeFromPostgres(pgErr.Code) + } + + return CodeInternal +} + +var ( + httpStatusByCode = map[Code]int{ + CodeBadRequest: http.StatusBadRequest, + CodeUnauthorized: http.StatusUnauthorized, + CodeForbidden: http.StatusForbidden, + CodeNotFound: http.StatusNotFound, + CodeAlreadyExists: http.StatusConflict, + CodeConflict: http.StatusConflict, + CodePayloadTooLarge: http.StatusRequestEntityTooLarge, + CodeTooManyRequests: http.StatusTooManyRequests, + CodeValidation: http.StatusBadRequest, + CodeTimeout: http.StatusGatewayTimeout, + CodeInternal: http.StatusInternalServerError, + CodeServiceUnavailable: http.StatusServiceUnavailable, + CodeUnsupportedMediaType: http.StatusUnsupportedMediaType, + CodeUnavailable: http.StatusServiceUnavailable, + CodeOK: http.StatusOK, + CodeLocked: http.StatusLocked, + } + codeByHTTPStatus = map[int]Code{ + http.StatusBadRequest: CodeBadRequest, + http.StatusUnauthorized: CodeUnauthorized, + http.StatusForbidden: CodeForbidden, + http.StatusNotFound: CodeNotFound, + http.StatusConflict: CodeConflict, + http.StatusRequestEntityTooLarge: CodePayloadTooLarge, + http.StatusTooManyRequests: CodeTooManyRequests, + http.StatusUnprocessableEntity: CodeValidation, + http.StatusGatewayTimeout: CodeTimeout, + http.StatusInternalServerError: CodeInternal, + http.StatusServiceUnavailable: CodeServiceUnavailable, + http.StatusOK: CodeOK, + http.StatusLocked: CodeLocked, + http.StatusUnsupportedMediaType: CodeUnavailable, + } + grpcCodeByCode = map[Code]grpcCodes.Code{ + CodeBadRequest: grpcCodes.InvalidArgument, + CodePayloadTooLarge: grpcCodes.ResourceExhausted, + CodeValidation: grpcCodes.InvalidArgument, + CodeUnauthorized: grpcCodes.Unauthenticated, + CodeForbidden: grpcCodes.PermissionDenied, + CodeNotFound: grpcCodes.NotFound, + CodeAlreadyExists: grpcCodes.AlreadyExists, + CodeConflict: grpcCodes.Aborted, + CodeTooManyRequests: grpcCodes.ResourceExhausted, + CodeTimeout: grpcCodes.DeadlineExceeded, + CodeInternal: grpcCodes.Internal, + CodeServiceUnavailable: grpcCodes.Unavailable, + CodeOK: grpcCodes.OK, + CodeLocked: grpcCodes.FailedPrecondition, + } + codeByGRPCCode = map[grpcCodes.Code]Code{ + grpcCodes.InvalidArgument: CodeBadRequest, + grpcCodes.ResourceExhausted: CodeTooManyRequests, + grpcCodes.Unauthenticated: CodeUnauthorized, + grpcCodes.PermissionDenied: CodeForbidden, + grpcCodes.NotFound: CodeNotFound, + grpcCodes.AlreadyExists: CodeAlreadyExists, + grpcCodes.Aborted: CodeConflict, + grpcCodes.FailedPrecondition: CodeLocked, + grpcCodes.DeadlineExceeded: CodeTimeout, + grpcCodes.Internal: CodeInternal, + grpcCodes.Unimplemented: CodeInternal, + grpcCodes.Unavailable: CodeServiceUnavailable, + grpcCodes.OK: CodeOK, + } + postgresCodesByCode = map[Code][]string{ + CodeBadRequest: { + pgerrcode.InvalidTextRepresentation, + pgerrcode.InvalidParameterValue, + }, + CodeValidation: { + pgerrcode.NotNullViolation, + pgerrcode.CheckViolation, + pgerrcode.StringDataRightTruncationDataException, + pgerrcode.NumericValueOutOfRange, + }, + CodeNotFound: { + pgerrcode.ForeignKeyViolation, + }, + CodeAlreadyExists: { + pgerrcode.UniqueViolation, + }, + CodeConflict: { + pgerrcode.ExclusionViolation, + pgerrcode.SerializationFailure, + pgerrcode.DeadlockDetected, + }, + CodeLocked: { + pgerrcode.LockNotAvailable, + pgerrcode.ObjectInUse, + }, + CodeTooManyRequests: { + pgerrcode.TooManyConnections, + }, + CodeTimeout: { + pgerrcode.QueryCanceled, + }, + CodeInternal: { + pgerrcode.ConnectionException, + pgerrcode.ConnectionFailure, + pgerrcode.SQLClientUnableToEstablishSQLConnection, + pgerrcode.SQLServerRejectedEstablishmentOfSQLConnection, + pgerrcode.TransactionResolutionUnknown, + }, + CodeServiceUnavailable: { + pgerrcode.CannotConnectNow, + }, + } + codeByPostgresCode = buildCodeByPostgresCode(postgresCodesByCode) + codeByPgxError = map[error]Code{ + pgx.ErrNoRows: CodeNotFound, + pgx.ErrTxClosed: CodeInternal, + pgx.ErrTxCommitRollback: CodeConflict, + pgx.ErrTooManyRows: CodeInternal, + } +) + +func buildCodeByPostgresCode(src map[Code][]string) map[string]Code { + dst := make(map[string]Code, len(src)) + for code, pgCodes := range src { + for _, pgCode := range pgCodes { + dst[pgCode] = code + } + } + return dst +} diff --git a/service/internal/pkg/apperr/code_string.go b/service/internal/pkg/apperr/code_string.go new file mode 100644 index 0000000..be690bf --- /dev/null +++ b/service/internal/pkg/apperr/code_string.go @@ -0,0 +1,37 @@ +// Code generated by "stringer -type=Code -linecomment"; DO NOT EDIT. + +package apperr + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[CodeBadRequest-0] + _ = x[CodePayloadTooLarge-1] + _ = x[CodeValidation-2] + _ = x[CodeUnauthorized-3] + _ = x[CodeForbidden-4] + _ = x[CodeNotFound-5] + _ = x[CodeAlreadyExists-6] + _ = x[CodeConflict-7] + _ = x[CodeTooManyRequests-8] + _ = x[CodeTimeout-9] + _ = x[CodeInternal-10] + _ = x[CodeServiceUnavailable-11] + _ = x[CodeOK-12] + _ = x[CodeLocked-13] +} + +const _Code_name = "CodeBadRequestCodePayloadTooLargeCodeValidationCodeUnauthorizedCodeForbiddenCodeNotFoundCodeAlreadyExistsCodeConflictCodeTooManyRequestsCodeTimeoutCodeInternalCodeServiceUnavailableCodeOKCodeLocked" + +var _Code_index = [...]uint8{0, 14, 33, 47, 63, 76, 88, 105, 117, 136, 147, 159, 181, 187, 197} + +func (i Code) String() string { + idx := int(i) - 0 + if i < 0 || idx >= len(_Code_index)-1 { + return "Code(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Code_name[_Code_index[idx]:_Code_index[idx+1]] +} diff --git a/service/internal/pkg/apperr/error.go b/service/internal/pkg/apperr/error.go new file mode 100644 index 0000000..7b45fe0 --- /dev/null +++ b/service/internal/pkg/apperr/error.go @@ -0,0 +1,123 @@ +package apperr + +import ( + "errors" + "fmt" + + grpcCodes "google.golang.org/grpc/codes" +) + +type AppError struct { + err error + msg string + code Code + meta metadata +} + +func New(code Code, msg string) error { + return &AppError{ + msg: msg, + code: code, + meta: metadata{}, + } +} + +func Wrap(err error, code Code, msg string) error { + m := AppError{ + err: err, + msg: msg, + code: code, + meta: metadata{}, + } + return &m +} + +func WrapAppError(err error, msg string) error { + m := AppError{ + err: err, + msg: msg, + code: CodeInternal, + meta: metadata{}, + } + + if mistake := As(err); mistake != nil { + m.err = mistake + m.code = mistake.code + m.meta = mistake.meta + } + + return &m +} + +func As(err error) *AppError { + if mistakeErr, ok := errors.AsType[*AppError](err); ok { + return mistakeErr + } + return nil +} + +func CodeIs(err error, code Code) bool { + if appErr := As(err); appErr != nil && appErr.Code() == code { + return true + } + return false +} + +func (e *AppError) Error() string { + if e == nil { + return "" + } + + if e.err == nil { + if e.msg != "" { + return e.msg + } + return e.code.String() + } + + if e.msg == "" { + return e.err.Error() + } + + return fmt.Sprintf("%s: %s", e.msg, e.err.Error()) +} + +func (e *AppError) Unwrap() error { + if e == nil { + return nil + } + return e.err +} + +func (e *AppError) Code() Code { + if e == nil { + return CodeInternal + } + return e.code +} + +func (e *AppError) Message() string { + if e == nil { + return "" + } + return e.msg +} + +func (e *AppError) Cause() error { + if e == nil { + return nil + } + return e.err +} + +func (e *AppError) Metadata() Metadata { + return e.meta +} + +func (e *AppError) HTTPStatus() int { + return e.Code().HTTPStatus() +} + +func (e *AppError) GRPCCode() grpcCodes.Code { + return e.Code().GRPCCode() +} diff --git a/service/internal/pkg/apperr/meta.go b/service/internal/pkg/apperr/meta.go new file mode 100644 index 0000000..195666e --- /dev/null +++ b/service/internal/pkg/apperr/meta.go @@ -0,0 +1,42 @@ +package apperr + +import ( + "fmt" + "maps" + "strings" +) + +var _ Metadata = (*metadata)(nil) + +type Metadata interface { + Get(key metakey) string + Clone() map[metakey]string + CloneToAnyMap() map[string]any + String() string +} + +type metadata map[metakey]string + +func (m metadata) Get(key metakey) string { + return m[key] +} + +func (m metadata) Clone() map[metakey]string { + return maps.Clone(m) +} + +func (m metadata) CloneToAnyMap() map[string]any { + newM := make(map[string]any, len(m)) + for k, v := range m { + newM[k.String()] = v + } + return newM +} + +func (m metadata) String() string { + var b strings.Builder + for k, v := range m { + b.WriteString(fmt.Sprintf("%s: %s, ", k, v)) + } + return b.String() +} diff --git a/service/internal/pkg/apperr/metakey.go b/service/internal/pkg/apperr/metakey.go new file mode 100644 index 0000000..7dc091f --- /dev/null +++ b/service/internal/pkg/apperr/metakey.go @@ -0,0 +1,19 @@ +package apperr + +type metakey uint16 + +const ( + PgErrorCode metakey = iota + RedisErrorCode +) + +func (m metakey) String() string { + switch m { + case PgErrorCode: + return "pg_err_code" + case RedisErrorCode: + return "redis_err_code" + default: + return "unknown" + } +} diff --git a/service/internal/pkg/apperr/postgres.go b/service/internal/pkg/apperr/postgres.go new file mode 100644 index 0000000..9b82e2c --- /dev/null +++ b/service/internal/pkg/apperr/postgres.go @@ -0,0 +1,81 @@ +package apperr + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +func WrapPostgres(err error, msg string) error { + m := AppError{ + err: err, + msg: msg, + code: CodeInternal, + meta: metadata{}, + } + + m.handlePostgres() + + return &m +} + +func (e *AppError) handlePostgres() { + if e.err == nil { + e.code = CodeOK + return + } + + var networkError net.Error + var connectError *pgconn.ConnectError + if errors.Is(e.err, context.DeadlineExceeded) || errors.Is(e.err, io.EOF) || + errors.Is(e.err, io.ErrUnexpectedEOF) || errors.As(e.err, &networkError) || errors.As(e.err, &connectError) { + e.code = CodeServiceUnavailable + e.msg = "database unavailable" + return + } + if pgErr, ok := errors.AsType[*pgconn.PgError](e.err); ok { + if strings.HasPrefix(pgErr.Code, "08") || pgErr.Code == "57014" || + pgErr.Code == "57P01" || pgErr.Code == "57P02" || pgErr.Code == "57P03" || pgErr.Code == "53300" { + e.code = CodeServiceUnavailable + e.msg = "database unavailable" + return + } + e.meta[PgErrorCode] = pgErr.Code + e.code = CodeFromPostgres(pgErr.Code) + return + } + switch { + case errors.Is(e.err, pgx.ErrNoRows): + e.msg = fmt.Sprintf("not found: %s", e.msg) + e.code = CodeNotFound + return + case errors.Is(e.err, pgx.ErrTooManyRows): + e.msg = fmt.Sprintf("got too many rows: %s", e.msg) + e.code = CodeInternal + return + case errors.Is(e.err, pgx.ErrTxClosed): + e.msg = fmt.Sprintf("transaction is closed: %s", e.msg) + e.code = CodeInternal + return + case errors.Is(e.err, pgx.ErrTxCommitRollback): + e.msg = fmt.Sprintf("transaction commit resulted in rollback: %s", e.msg) + e.code = CodeInternal + return + } +} + +func (e *AppError) PgCode() string { + if e == nil { + return "" + } + if e.meta == nil { + return "" + } + return e.meta.Get(PgErrorCode) +} diff --git a/service/internal/pkg/apperr/redis.go b/service/internal/pkg/apperr/redis.go new file mode 100644 index 0000000..f2afe56 --- /dev/null +++ b/service/internal/pkg/apperr/redis.go @@ -0,0 +1,170 @@ +package apperr + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + + "github.com/redis/go-redis/v9" +) + +func WrapRedis(err error, msg string) error { + m := AppError{ + err: err, + msg: msg, + meta: metadata{}, + } + + m.handleRedis() + + return &m +} + +func CodeFromRedisError(err error) Code { + code, ok := codeFromRedisError(err, true, true) + if !ok { + return CodeInternal + } + return code +} + +func (e *AppError) handleRedis() { + if e.err == nil { + e.code = CodeOK + return + } + + if code := redisErrorCode(e.err); code != "" { + e.meta[RedisErrorCode] = code + } + + if errors.Is(e.err, redis.Nil) { + e.msg = fmt.Sprintf("not found: %s", e.msg) + } + + e.code = CodeFromRedisError(e.err) +} + +func (e *AppError) RedisCode() string { + if e == nil { + return "" + } + if e.meta == nil { + return "" + } + return e.meta.Get(RedisErrorCode) +} + +func codeFromRedisError(err error, includeTransport bool, includeProtocolString bool) (Code, bool) { + if err == nil { + return CodeInternal, false + } + + switch { + case errors.Is(err, redis.Nil): + return CodeNotFound, true + case errors.Is(err, redis.TxFailedErr): + return CodeConflict, true + case errors.Is(err, redis.ErrCrossSlot): + return CodeBadRequest, true + case errors.Is(err, redis.ErrPoolTimeout): + return CodeTimeout, true + case errors.Is(err, redis.ErrPoolExhausted): + return CodeTooManyRequests, true + case errors.Is(err, redis.ErrClosed): + return CodeServiceUnavailable, true + } + + if includeTransport { + if errors.Is(err, context.Canceled) || + errors.Is(err, context.DeadlineExceeded) { + return CodeTimeout, true + } + + if netErr, ok := errors.AsType[net.Error](err); ok { + if netErr.Timeout() { + return CodeTimeout, true + } + return CodeServiceUnavailable, true + } + } + + hasRedisError := isRedisError(err) + if !hasRedisError && !includeProtocolString { + return CodeInternal, false + } + + switch { + case redis.IsExecAbortError(err): + return CodeConflict, true + case redis.IsMaxClientsError(err): + return CodeTooManyRequests, true + case redis.IsLoadingError(err), + redis.IsReadOnlyError(err), + redis.IsClusterDownError(err), + redis.IsTryAgainError(err), + redis.IsMasterDownError(err), + redis.IsNoReplicasError(err), + redis.IsOOMError(err): + return CodeServiceUnavailable, true + case redis.IsAuthError(err), + redis.IsPermissionError(err): + return CodeInternal, true + } + + if _, ok := redis.IsMovedError(err); ok { + return CodeServiceUnavailable, true + } + if _, ok := redis.IsAskError(err); ok { + return CodeServiceUnavailable, true + } + + if hasRedisError { + return CodeInternal, true + } + + return CodeInternal, false +} + +func isRedisError(err error) bool { + if err == nil { + return false + } + + var redisErr redis.Error + return errors.As(err, &redisErr) +} + +func redisErrorCode(err error) string { + switch { + case err == nil: + return "" + case errors.Is(err, redis.Nil): + return "nil" + case errors.Is(err, redis.TxFailedErr): + return "tx_failed" + case errors.Is(err, redis.ErrCrossSlot): + return "CROSSSLOT" + case errors.Is(err, redis.ErrPoolTimeout): + return "pool_timeout" + case errors.Is(err, redis.ErrPoolExhausted): + return "pool_exhausted" + case errors.Is(err, redis.ErrClosed): + return "closed" + } + + var redisErr redis.Error + if !errors.As(err, &redisErr) { + return "" + } + + msg := strings.TrimSpace(redisErr.Error()) + msg = strings.TrimPrefix(msg, "ERR ") + parts := strings.Fields(msg) + if len(parts) == 0 { + return "" + } + return parts[0] +} diff --git a/service/internal/pkg/auth/jwtprovider/claims.go b/service/internal/pkg/auth/jwtprovider/claims.go new file mode 100644 index 0000000..bc8dfac --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/claims.go @@ -0,0 +1,145 @@ +package jwtprovider + +import ( + "fmt" + "log/slog" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +var _ Claims = (*jwtClaims)(nil) + +type Claims interface { + GetTopTokenID() (uuid.UUID, bool) + GetTopUserID() (uuid.UUID, bool) + GetTopRole() (role.Role, bool) + GetTopIsAccess() (bool, bool) + GetTopExpiresAt() (time.Time, bool) + GetTopIssuedAt() (time.Time, bool) +} + +type claimsParseFunc func(token *dto.Token) (*jwtClaims, error) + +type jwtClaims struct { + jwt.RegisteredClaims + + Nonce string `json:"nonce,omitempty"` + IsAccess bool `json:"is_access"` + Role role.Role `json:"role"` +} + +func (claims *jwtClaims) GetTopTokenID() (uuid.UUID, bool) { + parsed, err := uuid.Parse(claims.ID) + if err != nil { + return uuid.Nil, false + } + return parsed, true +} + +func (claims *jwtClaims) GetTopUserID() (uuid.UUID, bool) { + parsed, err := uuid.Parse(claims.Subject) + if err != nil { + return uuid.Nil, false + } + return parsed, true +} + +func (claims *jwtClaims) GetTopRole() (role.Role, bool) { + return claims.Role, true +} + +func (claims *jwtClaims) GetTopIsAccess() (bool, bool) { + return claims.IsAccess, true +} + +func (claims *jwtClaims) GetTopExpiresAt() (time.Time, bool) { + if claims.ExpiresAt == nil { + return time.Time{}, false + } + return claims.ExpiresAt.Time, true +} + +func (claims *jwtClaims) GetTopIssuedAt() (time.Time, bool) { + if claims.IssuedAt == nil { + return time.Time{}, false + } + return claims.IssuedAt.Time, true +} + +func (claims *jwtClaims) userData() (*dto.UserDataInToken, error) { + var u dto.UserDataInToken + + tokenID, err := uuid.Parse(claims.ID) + if err != nil { + return nil, fmt.Errorf("failed to parse token id: %w", err) + } + u.TokenID = tokenID + + userID, err := uuid.Parse(claims.Subject) + if err != nil { + return nil, fmt.Errorf("failed to parse user id from token %q: %w", claims.Subject, err) + } + u.UserID = userID + + u.Role = claims.Role + u.IsAccess = claims.IsAccess + + if claims.ExpiresAt != nil { + u.ExpiresAt = claims.ExpiresAt.Time + } + + if claims.IssuedAt != nil { + u.IssuedAt = claims.IssuedAt.Time + } + + return &u, nil +} + +func (p *provider) parseClaimsFromProtectedJWT(token *dto.Token) (*jwtClaims, error) { + if token == nil { + return nil, ErrTokenIsMissing + } + + claims := &jwtClaims{} + + parsed, err := jwt.ParseWithClaims( + token.Payload, + claims, + p.readKeyFunc, + jwt.WithValidMethods([]string{jwt.SigningMethodRS256.Alg()}), + jwt.WithExpirationRequired(), + jwt.WithIssuedAt(), + ) + if err != nil { + slog.Error("failed to parse token with claims", "error", err) + return nil, ErrInvalidToken + } + + if !parsed.Valid { + slog.Error("failed to parse token: provided invalid token", "valid", parsed.Valid) + return nil, ErrInvalidToken + } + + return claims, nil +} + +func (p *provider) parseClaimsFromExposedJWT(token *dto.Token) (*jwtClaims, error) { + if token == nil { + return nil, ErrTokenIsMissing + } + + claims := &jwtClaims{} + + _, _, err := new(jwt.Parser).ParseUnverified(token.Payload, claims) + if err != nil { + slog.Error("failed to parse token: provided invalid token") + return nil, ErrInvalidToken + } + + return claims, nil +} diff --git a/service/internal/pkg/auth/jwtprovider/cookie.go b/service/internal/pkg/auth/jwtprovider/cookie.go new file mode 100644 index 0000000..8609585 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/cookie.go @@ -0,0 +1,84 @@ +package jwtprovider + +import ( + "net/http" +) + +const ( + accessTokenCookieKey = "auth-access-token" + refreshTokenCookieKey = "auth-refresh-token" + expiresAtCookieKey = "auth-exp" +) + +type CookieBuilderFabric struct { + fabric func() http.Cookie +} + +func newCookieBuilder( + domain string, + path string, + secure bool, + sameSite http.SameSite, +) *CookieBuilderFabric { + return &CookieBuilderFabric{ + fabric: newCookieFabric(domain, path, secure, sameSite), + } +} + +func (b *CookieBuilderFabric) Builder() *CookieBuilder { + return &CookieBuilder{ + cookie: b.fabric(), + } +} + +func newCookieFabric( + _ string, + path string, + secure bool, + sameSite http.SameSite) func() http.Cookie { + return func() http.Cookie { + return http.Cookie{ + Path: path, + Secure: secure, + SameSite: sameSite, + } + } +} + +type CookieBuilder struct { + cookie http.Cookie +} + +func (b *CookieBuilder) AccessName() *CookieBuilder { + b.cookie.Name = accessTokenCookieKey + return b +} + +func (b *CookieBuilder) ExpiresName() *CookieBuilder { + b.cookie.Name = expiresAtCookieKey + return b +} + +func (b *CookieBuilder) RefreshName() *CookieBuilder { + b.cookie.Name = refreshTokenCookieKey + return b +} + +func (b *CookieBuilder) Value(value string) *CookieBuilder { + b.cookie.Value = value + return b +} + +func (b *CookieBuilder) HTTPOnly() *CookieBuilder { + b.cookie.HttpOnly = true + return b +} + +func (b *CookieBuilder) MaxAge(maxAge int) *CookieBuilder { + b.cookie.MaxAge = maxAge + return b +} + +func (b *CookieBuilder) Build() *http.Cookie { + return new(b.cookie) +} diff --git a/service/internal/pkg/auth/jwtprovider/error.go b/service/internal/pkg/auth/jwtprovider/error.go new file mode 100644 index 0000000..6fde812 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/error.go @@ -0,0 +1,18 @@ +package jwtprovider + +import ( + "errors" +) + +var ( + ErrDecode = errors.New("failed to decode argument") + ErrNotFound = errors.New("not found") + ErrUnexpectedAlgorithm = errors.New("unexpected signing algorithm") + ErrTokenIsMissing = errors.New("token is missing") + ErrInvalidToken = errors.New("provided invalid token") + ErrInvalidUserDataInToken = errors.New("invalid user data in token") + ErrInvalidTokenClaims = errors.New("provided invalid token claims") + ErrInvalidBirthDateFormat = errors.New("provided invalid birth date format") + ErrTokenExpired = errors.New("token is expired") + ErrFailedToSign = errors.New("failed to sign jwt") +) diff --git a/service/internal/pkg/auth/jwtprovider/interface.go b/service/internal/pkg/auth/jwtprovider/interface.go new file mode 100644 index 0000000..ced83ee --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/interface.go @@ -0,0 +1,26 @@ +package jwtprovider + +import ( + "net/http" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +type Provider interface { + Parse(token *dto.Token) (*dto.UserDataInToken, error) + CreateRefreshAndAccessTokensForUser(userID uuid.UUID, role role.Role) (*dto.Tokens, error) + ParseVerified(token *dto.Token) (*dto.UserDataInToken, error) + ParseUnverified(token *dto.Token) (*dto.UserDataInToken, error) + ValidateUserDataInToken(u *dto.UserDataInToken) error +} + +type Cookie interface { + Enabled() bool + Set(w http.ResponseWriter, tokens *dto.Tokens) + Clear(w http.ResponseWriter) + AccessToken(r *http.Request) *dto.AccessToken + RefreshToken(r *http.Request) *dto.RefreshToken +} diff --git a/service/internal/pkg/auth/jwtprovider/manager.go b/service/internal/pkg/auth/jwtprovider/manager.go new file mode 100644 index 0000000..baeda31 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/manager.go @@ -0,0 +1,199 @@ +package jwtprovider + +import ( + "fmt" + "net/http" + "strconv" + "time" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type manager struct { + enabled bool + cookieFabric *CookieBuilderFabric + prv Provider +} + +func NewCookieManager(cfg *config.Config, provider Provider) Cookie { + m := &manager{ + //enabled: cfg.Cookies.Enabled, + //cookieFabric: newCookieBuilder( + // cfg.Cookies.Domain, + // cfg.Cookies.Path, + // cfg.Cookies.Secure, + // cfg.Cookies.SameSite, + //), + prv: provider, + } + + if !m.enabled { + return m + } + + if err := m.cookieFabric. + Builder(). + AccessName(). + Value("test"). + HTTPOnly(). + MaxAge(0). + Build(). + Valid(); err != nil { + panic(fmt.Errorf("cookie validation failed: %w", err)) + } + + return m +} + +func (m *manager) Enabled() bool { + return m.enabled +} + +func (m *manager) Set(w http.ResponseWriter, tokens *dto.Tokens) { + if !m.enabled || tokens == nil { + return + } + + if tokens.AccessToken.Payload != "" { + m.setAccess(tokens.AccessToken, w) + } + + if tokens.RefreshToken.Payload != "" { + m.setRefresh(tokens.RefreshToken, w) + } +} + +func (m *manager) Clear(w http.ResponseWriter) { + if !m.enabled { + return + } + + for _, name := range []string{accessTokenCookieKey, refreshTokenCookieKey, expiresAtCookieKey} { + b := m.cookieFabric. + Builder(). + MaxAge(-1) + + switch name { + case expiresAtCookieKey: + b.ExpiresName() + case refreshTokenCookieKey: + b.RefreshName(). + HTTPOnly() + case accessTokenCookieKey: + b.AccessName(). + HTTPOnly() + } + http.SetCookie(w, b.Build()) + } +} + +func (m *manager) AccessToken(r *http.Request) *dto.AccessToken { + if !m.enabled { + return nil + } + + c, err := r.Cookie(accessTokenCookieKey) + if err != nil { + return nil + } + + return &dto.AccessToken{ + Payload: c.Value, + } +} + +func (m *manager) RefreshToken(r *http.Request) *dto.RefreshToken { + if !m.enabled { + return nil + } + + c, err := r.Cookie(refreshTokenCookieKey) + if err != nil { + return nil + } + + return &dto.RefreshToken{ + Payload: c.Value, + } +} + +func (m *manager) setAccess(token dto.AccessToken, w http.ResponseWriter) { + accessTTL := m.tokenTTL(token.Token()) + if accessTTL < 0 { + http.SetCookie(w, m.cookieFabric. + Builder(). + AccessName(). + HTTPOnly(). + MaxAge(-1). + Build(), + ) + http.SetCookie(w, m.cookieFabric. + Builder(). + ExpiresName(). + MaxAge(-1). + Build(), + ) + } else if tokenExpiresAt := m.tokenExpiry(token.Token()); !tokenExpiresAt.IsZero() { + http.SetCookie(w, m.cookieFabric. + Builder(). + AccessName(). + Value(token.Payload). + HTTPOnly(). + MaxAge(accessTTL). + Build(), + ) + http.SetCookie(w, m.cookieFabric. + Builder(). + ExpiresName(). + Value(strconv.FormatInt(tokenExpiresAt.Unix(), 10)). + MaxAge(accessTTL). + Build(), + ) + } +} + +func (m *manager) setRefresh(token dto.RefreshToken, w http.ResponseWriter) { + refreshTTL := m.tokenTTL(token.Token()) + if refreshTTL < 0 { + http.SetCookie(w, m.cookieFabric. + Builder(). + RefreshName(). + HTTPOnly(). + MaxAge(-1). + Build(), + ) + } else { + http.SetCookie(w, m.cookieFabric. + Builder(). + RefreshName(). + Value(token.Payload). + HTTPOnly(). + MaxAge(refreshTTL). + Build(), + ) + } +} + +func (m *manager) tokenTTL(token *dto.Token) int { + expiresAt := m.tokenExpiry(token) + + if expiresAt.IsZero() { + return -1 + } + + ttl := int(time.Until(expiresAt).Seconds()) + if ttl <= 0 { + return -1 + } + + return ttl +} + +func (m *manager) tokenExpiry(token *dto.Token) time.Time { + data, err := m.prv.ParseUnverified(token) + if err != nil { + return time.Time{} + } + return data.ExpiresAt +} diff --git a/service/internal/pkg/auth/jwtprovider/provider.go b/service/internal/pkg/auth/jwtprovider/provider.go new file mode 100644 index 0000000..a387854 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/provider.go @@ -0,0 +1,187 @@ +package jwtprovider + +import ( + "crypto/rand" + "crypto/rsa" + "encoding/base64" + "fmt" + "log/slog" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +var ( + _ Provider = (*provider)(nil) +) + +type provider struct { + accessLifetime time.Duration + refreshLifetime time.Duration + publicKey *rsa.PublicKey + privateKey *rsa.PrivateKey + parseFunc claimsParseFunc +} + +func New(cfg *config.Config) Provider { + p := provider{ + accessLifetime: cfg.JWT.AccessTokenLifetime, + refreshLifetime: cfg.JWT.RefreshTokenLifetime, + } + p.parseFunc = p.parseClaimsFromProtectedJWT + + if cfg.Application.Environment == environment.ApplicationEnvironmentDev { + p.parseFunc = p.parseClaimsFromExposedJWT + } + + var err error + if p.publicKey, err = readPublicKeyFromFile(cfg.JWT.PublicKeyPath); err != nil { + slog.Error("failed to read public key", "error", err) + panic(fmt.Errorf("failed to read public key: %w", err)) + } + if p.privateKey, err = readPrivateKeyFromFile(cfg.JWT.PrivateKeyPath); err != nil { + slog.Error("failed to read private key", "error", err) + panic(fmt.Errorf("failed to read private key: %w", err)) + } + + return &p +} + +func (p *provider) Parse(token *dto.Token) (*dto.UserDataInToken, error) { + claims, err := p.parseFunc(token) + if err != nil { + return nil, err + } + return claims.userData() +} + +func (p *provider) ParseUnverified(t *dto.Token) (*dto.UserDataInToken, error) { + if t == nil { + return nil, ErrTokenIsMissing + } + + claims, err := p.parseClaimsFromExposedJWT(t) + if err != nil { + return nil, err + } + + return claims.userData() +} + +func (p *provider) ParseVerified(t *dto.Token) (*dto.UserDataInToken, error) { + if t == nil { + return nil, ErrTokenIsMissing + } + claims, err := p.parseClaimsFromProtectedJWT(t) + if err != nil { + return nil, fmt.Errorf("failed to parse token: %w", err) + } + + return claims.userData() +} + +func (p *provider) CreateAccessTokenForUser(userID uuid.UUID, role role.Role) (*dto.AccessToken, error) { + t, err := p.createTokenForUser(userID, true, role) + if err != nil { + return nil, err + } + return &dto.AccessToken{ + Payload: t.Payload, + }, nil +} + +func (p *provider) CreateRefreshTokenForUser(userID uuid.UUID, role role.Role) (*dto.RefreshToken, error) { + t, err := p.createTokenForUser(userID, false, role) + if err != nil { + return nil, err + } + return &dto.RefreshToken{ + Payload: t.Payload, + }, nil +} + +func (p *provider) CreateRefreshAndAccessTokensForUser(userID uuid.UUID, role role.Role) (*dto.Tokens, error) { + access, err := p.CreateAccessTokenForUser(userID, role) + if err != nil { + return nil, err + } + refresh, err := p.CreateRefreshTokenForUser(userID, role) + if err != nil { + return nil, err + } + return &dto.Tokens{ + AccessToken: *access, + RefreshToken: *refresh, + }, nil +} + +func (p *provider) createTokenForUser(userID uuid.UUID, isAccess bool, role role.Role) (*dto.Token, error) { + now := time.Now() + var tokenExpiresAt time.Time + if isAccess { + tokenExpiresAt = now.Add(p.accessLifetime) + } else { + tokenExpiresAt = now.Add(p.refreshLifetime) + } + var nonce [32]byte + if _, err := rand.Read(nonce[:]); err != nil { + return nil, fmt.Errorf("generate token nonce: %w", err) + } + claims := &jwtClaims{ + Nonce: base64.RawURLEncoding.EncodeToString(nonce[:]), + RegisteredClaims: jwt.RegisteredClaims{ + Subject: userID.String(), + ExpiresAt: jwt.NewNumericDate(tokenExpiresAt), + NotBefore: jwt.NewNumericDate(now), + IssuedAt: jwt.NewNumericDate(now), + ID: uuid.NewString(), + }, + IsAccess: isAccess, + Role: role, + } + t := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + + payload, err := t.SignedString(p.privateKey) + if err != nil { + slog.Error("failed to sign jwt", "error", err) + return nil, ErrFailedToSign + } + return &dto.Token{Payload: payload}, nil +} + +// TODO refactor maybe delete +func (p *provider) ValidateUserDataInToken(u *dto.UserDataInToken) error { + if u == nil { + return ErrInvalidUserDataInToken + } + + switch { + case u.TokenID == uuid.Nil: + return fmt.Errorf("%w: unexpected nil token id", ErrInvalidUserDataInToken) + case u.UserID == uuid.Nil: + return fmt.Errorf("%w: unexpected nil user id", ErrInvalidUserDataInToken) + case u.IssuedAt.IsZero(): + return fmt.Errorf("%w: unexpected zero issued at", ErrInvalidUserDataInToken) + case u.Email != nil && *u.Email == "": + return fmt.Errorf("%w: unexpected empty email", ErrInvalidUserDataInToken) + case u.PhoneNumber != nil && *u.PhoneNumber == "": + return fmt.Errorf("%w: unexpected empty phone number", ErrInvalidUserDataInToken) + case u.ExpiresAt.IsZero(): + return fmt.Errorf("%w: unexpected zero expires at", ErrInvalidUserDataInToken) + } + + return nil +} + +func (p *provider) readKeyFunc(token *jwt.Token) (interface{}, error) { + if token.Method != jwt.SigningMethodRS256 { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return p.publicKey, nil +} diff --git a/service/internal/pkg/auth/jwtprovider/readers.go b/service/internal/pkg/auth/jwtprovider/readers.go new file mode 100644 index 0000000..f2c3ae5 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/readers.go @@ -0,0 +1,35 @@ +package jwtprovider + +import ( + "crypto/rsa" + "fmt" + "os" + + "github.com/golang-jwt/jwt/v5" +) + +func readPrivateKeyFromFile(path string) (*rsa.PrivateKey, error) { + key, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read public key: %w", err) + } + + privateKey, err := jwt.ParseRSAPrivateKeyFromPEM(key) + if err != nil { + return nil, fmt.Errorf("failed to parse private key: %w", err) + } + return privateKey, nil +} + +func readPublicKeyFromFile(path string) (*rsa.PublicKey, error) { + key, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read public key: %w", err) + } + + privateKey, err := jwt.ParseRSAPublicKeyFromPEM(key) + if err != nil { + return nil, fmt.Errorf("failed to parse private key: %w", err) + } + return privateKey, nil +} diff --git a/service/internal/pkg/cache/redis.go b/service/internal/pkg/cache/redis.go new file mode 100644 index 0000000..91f90eb --- /dev/null +++ b/service/internal/pkg/cache/redis.go @@ -0,0 +1,43 @@ +package cache + +import ( + "context" + "time" + + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/mtusi2026/internal/config" +) + +const defaultRedisPort = 6379 + +// OperationTimeout bounds each optional cache operation independently of the request. +const OperationTimeout = 300 * time.Millisecond + +type Client struct { + *redis.Client +} + +func New(cfg *config.Config) *Client { + if cfg.Redis.Port == 0 { + cfg.Redis.Port = defaultRedisPort + } + + return &Client{ + Client: redis.NewClient(cfg.Redis.BuildRedisOptions()), + } +} + +func (c *Client) Ping(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, OperationTimeout) + defer cancel() + return c.Client.Ping(ctx).Err() +} + +func (c *Client) Start(ctx context.Context) error { + return c.Ping(ctx) +} + +func (c *Client) Stop(_ context.Context) error { + return c.Client.Close() +} diff --git a/service/internal/pkg/encrypting/argon2id.go b/service/internal/pkg/encrypting/argon2id.go new file mode 100644 index 0000000..3a52f9b --- /dev/null +++ b/service/internal/pkg/encrypting/argon2id.go @@ -0,0 +1,71 @@ +package encrypting + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "fmt" + "strings" + + "golang.org/x/crypto/argon2" +) + +const ( + argon2idMemory = 64 * 1024 // KiB + argon2idTime = 3 + argon2idThreads = 4 + argon2idSaltLen = 16 + argon2idKeyLen = 32 + // Only this bounded profile is accepted; stored parameters cannot request arbitrary resources. + argon2idPrefix = "$argon2id$v=19$m=65536,t=3,p=4$" +) + +func (e *encryptor) EncryptSecretUsingArgon2id(ctx context.Context, secret string) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + var salt [argon2idSaltLen]byte + if _, err := rand.Read(salt[:]); err != nil { + return "", fmt.Errorf("%w: %w", ErrFailedToEncryptPassword, err) + } + key := argon2.IDKey([]byte(secret), salt[:], argon2idTime, argon2idMemory, argon2idThreads, argon2idKeyLen) + if err := ctx.Err(); err != nil { + return "", err + } + return argon2idPrefix + base64.RawStdEncoding.EncodeToString(salt[:]) + "$" + base64.RawStdEncoding.EncodeToString(key), nil +} + +func (e *encryptor) CompareHashAndSecretUsingArgon2id(ctx context.Context, hash, secret string) error { + if err := ctx.Err(); err != nil { + return err + } + encoding := base64.RawStdEncoding.Strict() + if len(hash) != len(argon2idPrefix)+encoding.EncodedLen(argon2idSaltLen)+1+encoding.EncodedLen(argon2idKeyLen) { + return ErrFailedToComparePasswords + } + payload, ok := strings.CutPrefix(hash, argon2idPrefix) + if !ok { + return ErrFailedToComparePasswords + } + saltText, keyText, ok := strings.Cut(payload, "$") + if !ok { + return ErrFailedToComparePasswords + } + salt, err := encoding.DecodeString(saltText) + if err != nil || len(salt) != argon2idSaltLen { + return ErrFailedToComparePasswords + } + storedKey, err := encoding.DecodeString(keyText) + if err != nil || len(storedKey) != argon2idKeyLen { + return ErrFailedToComparePasswords + } + key := argon2.IDKey([]byte(secret), salt, argon2idTime, argon2idMemory, argon2idThreads, argon2idKeyLen) + if err = ctx.Err(); err != nil { + return err + } + if subtle.ConstantTimeCompare(storedKey, key) != 1 { + return ErrFailedToComparePasswords + } + return nil +} diff --git a/service/internal/pkg/encrypting/encryptor.go b/service/internal/pkg/encrypting/encryptor.go new file mode 100644 index 0000000..1495a5b --- /dev/null +++ b/service/internal/pkg/encrypting/encryptor.go @@ -0,0 +1,72 @@ +package encrypting + +import ( + "context" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "strings" + + "golang.org/x/crypto/bcrypt" +) + +type encryptor struct { + cost int + generateFromPasswordFunc func([]byte, int) ([]byte, error) + compareHashAndPasswordFunc func([]byte, []byte) error +} + +func New() Encryptor { + return &encryptor{ + cost: bcrypt.DefaultCost, + generateFromPasswordFunc: bcrypt.GenerateFromPassword, + compareHashAndPasswordFunc: bcrypt.CompareHashAndPassword, + } +} + +func (e *encryptor) EncryptSecret(_ context.Context, secret string) (string, error) { + res, err := e.generateFromPasswordFunc([]byte(secret), e.cost) + if err != nil { + return "", ErrFailedToEncryptPassword + } + return string(res), nil +} + +// EncryptSecretUsingSum256 hashes a high-entropy session token, not a password. +func (e *encryptor) EncryptSecretUsingSum256(ctx context.Context, secret string) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + sum := sha256.Sum256([]byte(secret)) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + +func (e *encryptor) CompareHashAndSecret(_ context.Context, hash, password string) error { + err := e.compareHashAndPasswordFunc([]byte(hash), []byte(password)) + if err != nil { + return ErrFailedToComparePasswords + } + return nil +} + +func (e *encryptor) CompareHashAndSecretUsingSum256(ctx context.Context, hash, secret string) error { + if err := ctx.Err(); err != nil { + return err + } + sum := sha256.Sum256([]byte(secret)) + if payload, ok := strings.CutPrefix(hash, "sha256:"); ok { + stored, err := hex.DecodeString(payload) + if err != nil || len(stored) != sha256.Size || subtle.ConstantTimeCompare(stored, sum[:]) != 1 { + return ErrFailedToComparePasswords + } + return nil + } + // Preserve existing lab2/lab3 sessions across a rolling deployment. + // Passwords continue to use Argon2id; only old random-token fingerprints use this fallback. + if strings.HasPrefix(hash, "$2") { + if err := e.compareHashAndPasswordFunc([]byte(hash), []byte(hex.EncodeToString(sum[:]))); err == nil { + return nil + } + } + return ErrFailedToComparePasswords +} diff --git a/service/internal/pkg/encrypting/error.go b/service/internal/pkg/encrypting/error.go new file mode 100644 index 0000000..ffbc22f --- /dev/null +++ b/service/internal/pkg/encrypting/error.go @@ -0,0 +1,8 @@ +package encrypting + +import "errors" + +var ( + ErrFailedToEncryptPassword = errors.New("failed to encrypt password") + ErrFailedToComparePasswords = errors.New("failed to compare passwords") +) diff --git a/service/internal/pkg/encrypting/interface.go b/service/internal/pkg/encrypting/interface.go new file mode 100644 index 0000000..4f60f24 --- /dev/null +++ b/service/internal/pkg/encrypting/interface.go @@ -0,0 +1,12 @@ +package encrypting + +import "context" + +type Encryptor interface { + EncryptSecretUsingArgon2id(ctx context.Context, secret string) (string, error) + CompareHashAndSecretUsingArgon2id(ctx context.Context, hash, secret string) error + EncryptSecret(ctx context.Context, secret string) (string, error) + EncryptSecretUsingSum256(_ context.Context, secret string) (string, error) + CompareHashAndSecret(ctx context.Context, hash, secret string) error + CompareHashAndSecretUsingSum256(ctx context.Context, hash, secret string) error +} diff --git a/service/internal/pkg/encrypting/token_test.go b/service/internal/pkg/encrypting/token_test.go new file mode 100644 index 0000000..20934b1 --- /dev/null +++ b/service/internal/pkg/encrypting/token_test.go @@ -0,0 +1,43 @@ +package encrypting + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "golang.org/x/crypto/bcrypt" + "testing" +) + +func TestSessionFingerprints(t *testing.T) { + e := New() + ctx := context.Background() + token := "synthetic-high-entropy-session-value" + hash, err := e.EncryptSecretUsingSum256(ctx, token) + if err != nil { + t.Fatal(err) + } + if err = e.CompareHashAndSecretUsingSum256(ctx, hash, token); err != nil { + t.Fatal(err) + } + for _, bad := range []string{"sha256:00", "sha256:invalid", "unknown"} { + if e.CompareHashAndSecretUsingSum256(ctx, bad, token) == nil { + t.Fatal("accepted malformed fingerprint") + } + } + if e.CompareHashAndSecretUsingSum256(ctx, hash, token+"x") == nil { + t.Fatal("accepted wrong token") + } + sum := sha256.Sum256([]byte(token)) + legacy, err := bcrypt.GenerateFromPassword([]byte(hex.EncodeToString(sum[:])), bcrypt.MinCost) + if err != nil { + t.Fatal(err) + } + if err = e.CompareHashAndSecretUsingSum256(ctx, string(legacy), token); err != nil { + t.Fatal("legacy session rejected", err) + } + canceled, cancel := context.WithCancel(ctx) + cancel() + if e.CompareHashAndSecretUsingSum256(canceled, hash, token) == nil { + t.Fatal("ignored cancellation") + } +} diff --git a/service/internal/pkg/fault/fault.go b/service/internal/pkg/fault/fault.go new file mode 100644 index 0000000..0d5af15 --- /dev/null +++ b/service/internal/pkg/fault/fault.go @@ -0,0 +1,64 @@ +// Package fault implements explicit, file-controlled barriers for the isolated lab. +package fault + +import ( + "context" + "errors" + "github.com/google/uuid" + "os" + "path/filepath" + "strings" + "time" +) + +func Hit(ctx context.Context, point string, id uuid.UUID) { + if os.Getenv("LAB5_FAULTS") != "1" { + return + } + p := filepath.Join("/faults", point+"."+id.String()) + b, e := os.ReadFile(p) + if e != nil { + p = filepath.Join("/faults", point+".all") + b, e = os.ReadFile(p) + if e != nil { + return + } + } + hit, e := os.OpenFile(p+".hit", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + if e != nil { + return + } + _, _ = hit.WriteString(id.String() + "\n") + _ = hit.Close() + switch strings.TrimSpace(string(b)) { + case "crash": + _ = os.Remove(p) + os.Exit(86) + case "pause": + for { + if _, e = os.Stat(p); os.IsNotExist(e) { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(50 * time.Millisecond): + } + } + } +} + +func Error(point string, id uuid.UUID) error { + if os.Getenv("LAB5_FAULTS") != "1" { + return nil + } + b, e := os.ReadFile(filepath.Join("/faults", point+"."+id.String())) + if e != nil { + return nil + } + v := strings.TrimSpace(string(b)) + if v == "temporary" || v == "permanent" { + return errors.New(v) + } + return nil +} diff --git a/service/internal/pkg/imageprocessor/image.go b/service/internal/pkg/imageprocessor/image.go new file mode 100644 index 0000000..b87678c --- /dev/null +++ b/service/internal/pkg/imageprocessor/image.go @@ -0,0 +1,83 @@ +package imageprocessor + +import ( + "context" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "os" + "os/exec" + "path/filepath" + "time" +) + +func run(ctx context.Context, args ...string) error { + cmd := exec.CommandContext(ctx, "magick", args...) + cmd.Env = append(os.Environ(), "MAGICK_MEMORY_LIMIT=256MiB", "MAGICK_MAP_LIMIT=0", "MAGICK_DISK_LIMIT=256MiB", "MAGICK_THREAD_LIMIT=1", "MAGICK_TIME_LIMIT=60") + cmd.WaitDelay = time.Second + if err := cmd.Run(); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("image conversion failed: %w", err) + } + return nil +} +func Validate(ctx context.Context, path string) error { + return run(ctx, "-regard-warnings", path, "null:") +} + +type Result struct { + Name, Path, MIME string + Width, Height int +} + +func fit(w, h, limit int) (int, int) { + if w <= limit && h <= limit { + return w, h + } + if w >= h { + return limit, max(1, h*limit/w) + } + return max(1, w*limit/h), limit +} +func Process(ctx context.Context, source, dir string, version int) ([]Result, error) { + normalized := filepath.Join(dir, "normalized.png") + if e := run(ctx, source, "-auto-orient", "-background", "white", "-alpha", "remove", "-alpha", "off", "-strip", "png:"+normalized); e != nil { + return nil, e + } + f, e := os.Open(normalized) + if e != nil { + return nil, e + } + cfg, _, e := image.DecodeConfig(f) + f.Close() + if e != nil { + return nil, e + } + tw, th := fit(cfg.Width, cfg.Height, 256) + ow, oh := fit(cfg.Width, cfg.Height, 1280) + out := []Result{{"thumbnail", filepath.Join(dir, "thumbnail.jpg"), "image/jpeg", tw, th}, {"crop", filepath.Join(dir, "crop.jpg"), "image/jpeg", 128, 128}, {"optimized", filepath.Join(dir, "optimized.webp"), "image/webp", ow, oh}} + for _, v := range out { + args := []string{normalized} + if v.Name == "crop" { + side := min(cfg.Width, cfg.Height) + args = append(args, "-gravity", "center", "-crop", fmt.Sprintf("%dx%d+0+0", side, side), "+repage") + } + args = append(args, "-resize", fmt.Sprintf("%dx%d!", v.Width, v.Height), "-strip", "-quality") + quality := "85" + if v.Name == "optimized" && version == 1 { + quality = "80" + } + coder := "jpeg:" + if v.Name == "optimized" { + coder = "webp:" + } + args = append(args, quality, coder+v.Path) + if e = run(ctx, args...); e != nil { + return nil, e + } + } + return out, nil +} diff --git a/service/internal/pkg/migrator/tern.go b/service/internal/pkg/migrator/tern.go new file mode 100644 index 0000000..493fce6 --- /dev/null +++ b/service/internal/pkg/migrator/tern.go @@ -0,0 +1,164 @@ +package migrator + +import ( + "context" + "io/fs" + "log/slog" + + "github.com/jackc/pgx/v5" + "github.com/jackc/tern/v2/migrate" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/migrations" +) + +type Migrator interface { + MigrateUp(ctx context.Context) error + MigrateDown(ctx context.Context) error + MigrateTo(ctx context.Context, version int32) error + GetVersion(ctx context.Context) (int32, error) + Configure(ctx context.Context) error + Close(ctx context.Context) error + Start(ctx context.Context) error + Stop(ctx context.Context) error +} + +type migrator struct { + cfg *config.Config + conn *pgx.Conn + migrator *migrate.Migrator + migrations fs.FS + configured bool + closed bool +} + +func New(cfg *config.Config) Migrator { + return &migrator{ + cfg: cfg, + conn: nil, + migrator: nil, + migrations: migrations.Migrations, + configured: false, + closed: false, + } +} + +func (m *migrator) MigrateUp(ctx context.Context) error { + if !m.configured { + return apperr.New(apperr.CodeInternal, "migrator was not configured") + } + return m.migrator.Migrate(ctx) +} + +func (m *migrator) MigrateDown(ctx context.Context) error { + if !m.configured { + return apperr.New(apperr.CodeInternal, "migrator was not configured") + } + return m.migrator.MigrateTo(ctx, 0) +} + +func (m *migrator) MigrateTo(ctx context.Context, version int32) error { + if !m.configured { + return apperr.New(apperr.CodeInternal, "migrator was not configured") + } + return m.migrator.MigrateTo(ctx, version) +} + +func (m *migrator) GetVersion(ctx context.Context) (int32, error) { + if !m.configured { + return -1, apperr.New(apperr.CodeInternal, "migrator was not configured") + } + return m.migrator.GetCurrentVersion(ctx) +} + +func (m *migrator) Configure(ctx context.Context) error { + if err := m.initConn(ctx); err != nil { + return err + } + if err := m.initMigrator(ctx); err != nil { + return err + } + if err := m.loadMigrations(); err != nil { + return err + } + m.configured = true + return nil +} + +func (m *migrator) Close(ctx context.Context) error { + if m.conn == nil || m.closed { + return nil + } + if err := m.closeConn(ctx); err != nil { + return err + } + m.closed = true + return nil +} + +func (m *migrator) Start(ctx context.Context) error { + if !m.cfg.Postgres.GetRunMigrations() { + slog.Info("flag run migrations is disabled: skipping migrating") + return nil + } + + err := m.Configure(ctx) + if err != nil { + slog.Error("failed to configure migrator", "error", err) + return err + } + err = m.MigrateUp(ctx) + if err != nil { + slog.Error("failed to migrate database up", "error", err) + return err + } + + if err = m.Close(ctx); err != nil { + slog.Error("failed close connection to database", "error", err) + return err + } + + m.closed = true + + return nil +} + +func (m *migrator) Stop(ctx context.Context) error { + return m.Close(ctx) +} + +func (m *migrator) loadMigrations() (err error) { + err = m.migrator.LoadMigrations(m.migrations) + if err != nil { + slog.Error("failed to load migrations", "error", err) + return apperr.New(apperr.CodeInternal, "failed to load migrations") + } + slog.Info("successfully loaded migrations", "error", err) + return nil +} + +func (m *migrator) initConn(ctx context.Context) (err error) { + m.closed = false + m.conn, err = pgx.Connect(ctx, m.cfg.Postgres.GetURI()) + if err != nil { + slog.Error("failed to create connection to database", "error", err) + return apperr.New(apperr.CodeInternal, "failed to create connection to database") + } + slog.Info("connected to database", "db", m.cfg.Postgres.Database) + return nil +} + +func (m *migrator) initMigrator(ctx context.Context) (err error) { + m.migrator, err = migrate.NewMigrator(ctx, m.conn, m.cfg.Postgres.GetVersionTableName()) + if err != nil { + slog.Error("failed to initialize migrator", "error", err) + return apperr.New(apperr.CodeInternal, "failed to initialize migrator") + } + slog.Info("successfully initialized migrator") + return nil +} + +func (m *migrator) closeConn(ctx context.Context) error { + return m.conn.Close(ctx) +} diff --git a/service/internal/pkg/objectstore/s3.go b/service/internal/pkg/objectstore/s3.go new file mode 100644 index 0000000..1c34423 --- /dev/null +++ b/service/internal/pkg/objectstore/s3.go @@ -0,0 +1,97 @@ +package objectstore + +import ( + "context" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "go.opentelemetry.io/otel/trace" + "io" + "net/http" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + + "gitea.privcloud.tech/mtusi2026/internal/config" +) + +type Store struct { + Client *s3.Client + Bucket string +} + +func New(c *config.Attachment) *Store { + return &Store{ + Bucket: c.Bucket, + Client: s3.New( + s3.Options{ + Region: c.Region, + BaseEndpoint: aws.String(c.Endpoint), + UsePathStyle: true, + Credentials: credentials.NewStaticCredentialsProvider( + c.AccessKey, + c.SecretKey, + "", + ), + RetryMaxAttempts: 1, + HTTPClient: &http.Client{Timeout: 30 * time.Second}, + }, + ), + } +} + +func (s *Store) Put(ctx context.Context, key, mime string, r io.Reader, size int64) (err error) { + ctx, sp := telemetry.Start(ctx, "s3.put", trace.SpanKindClient) + defer func() { telemetry.End(sp, err) }() + + _, err = s.Client.PutObject(ctx, &s3.PutObjectInput{Bucket: &s.Bucket, Key: &key, Body: r, ContentLength: &size, ContentType: &mime}) + return err +} + +func (s *Store) Get(ctx context.Context, key string) (out *s3.GetObjectOutput, err error) { + ctx, sp := telemetry.Start(ctx, "s3.get", trace.SpanKindClient) + defer func() { telemetry.End(sp, err) }() + + return s.Client.GetObject(ctx, &s3.GetObjectInput{Bucket: &s.Bucket, Key: &key}) +} + +func (s *Store) Head(ctx context.Context, key string) (out *s3.HeadObjectOutput, err error) { + ctx, sp := telemetry.Start(ctx, "s3.head", trace.SpanKindClient) + defer func() { telemetry.End(sp, err) }() + + return s.Client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: &s.Bucket, Key: &key}) +} + +func (s *Store) Delete(ctx context.Context, key string) (err error) { + ctx, sp := telemetry.Start(ctx, "s3.delete", trace.SpanKindClient) + defer func() { telemetry.End(sp, err) }() + + _, err = s.Client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &s.Bucket, Key: &key}) + return err +} + +func (s *Store) Ping(ctx context.Context) (err error) { + ctx, sp := telemetry.Start(ctx, "s3.ping", trace.SpanKindClient) + defer func() { telemetry.End(sp, err) }() + + _, err = s.Client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &s.Bucket}) + return err +} + +func Missing(err error) bool { + var api smithy.APIError + return errors.As(err, &api) && (api.ErrorCode() == "NoSuchKey" || api.ErrorCode() == "NotFound" || api.ErrorCode() == "404") +} + +func (s *Store) List(ctx context.Context, token string) (out *s3.ListObjectsV2Output, err error) { + ctx, sp := telemetry.Start(ctx, "s3.list", trace.SpanKindClient) + defer func() { telemetry.End(sp, err) }() + + q := &s3.ListObjectsV2Input{Bucket: &s.Bucket, MaxKeys: aws.Int32(100)} + if token != "" { + q.ContinuationToken = &token + } + return s.Client.ListObjectsV2(ctx, q) +} diff --git a/service/internal/pkg/postgres/after-connect.go b/service/internal/pkg/postgres/after-connect.go new file mode 100644 index 0000000..eafcc63 --- /dev/null +++ b/service/internal/pkg/postgres/after-connect.go @@ -0,0 +1,25 @@ +package postgres + +import ( + "context" + + "github.com/jackc/pgx/v5" + pgxUUID "github.com/vgarvardt/pgx-google-uuid/v5" +) + +type AfterConnect func(ctx context.Context, conn *pgx.Conn) error + +func AddUUIDSupport(_ context.Context, conn *pgx.Conn) error { + pgxUUID.Register(conn.TypeMap()) + return nil +} +func WithEnumTypeSupport(enumName string) AfterConnect { + return func(ctx context.Context, conn *pgx.Conn) error { + dt, err := conn.LoadType(ctx, enumName) + if err != nil { + return err + } + conn.TypeMap().RegisterType(dt) + return nil + } +} diff --git a/service/internal/pkg/postgres/config.go b/service/internal/pkg/postgres/config.go new file mode 100644 index 0000000..b0df40a --- /dev/null +++ b/service/internal/pkg/postgres/config.go @@ -0,0 +1,7 @@ +package postgres + +type MigrationConfig interface { + GetURI() string + GetVersionTableName() string + GetRunMigrations() bool +} diff --git a/service/internal/pkg/postgres/err.go b/service/internal/pkg/postgres/err.go new file mode 100644 index 0000000..4380b24 --- /dev/null +++ b/service/internal/pkg/postgres/err.go @@ -0,0 +1,10 @@ +package postgres + +import ( + "errors" +) + +var ( + errCreatingPool = errors.New("failed to create postgres pool") + errParseConfig = errors.New("failed to parse postgres config") +) diff --git a/service/internal/pkg/postgres/interfaces.go b/service/internal/pkg/postgres/interfaces.go new file mode 100644 index 0000000..6afe65f --- /dev/null +++ b/service/internal/pkg/postgres/interfaces.go @@ -0,0 +1,15 @@ +package postgres + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type Conn interface { + SendBatch(ctx context.Context, b *pgx.Batch) pgx.BatchResults + Exec(ctx context.Context, sql string, arguments ...any) (commandTag pgconn.CommandTag, err error) + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} diff --git a/service/internal/pkg/postgres/postgres.go b/service/internal/pkg/postgres/postgres.go new file mode 100644 index 0000000..27fc125 --- /dev/null +++ b/service/internal/pkg/postgres/postgres.go @@ -0,0 +1,88 @@ +package postgres + +import ( + "context" + "log/slog" + "strconv" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/tracelog" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" +) + +type ConfigWithLog interface { + GetLogLevel() string +} + +type ConfigWithTimeouts interface { + GetConnectTimeout() time.Duration + GetQueryTimeout() time.Duration +} + +type ConfigWithMaxConn interface { + GetMaxConnections() int32 +} + +func getLevelFromConfig(cfg MigrationConfig) tracelog.LogLevel { + if logCfg, ok := cfg.(ConfigWithLog); ok { + lvl, _ := tracelog.LogLevelFromString(logCfg.GetLogLevel()) + return lvl + } + return tracelog.LogLevelError +} + +func addMaxConnectionsToCfg(cfg any, poolCfg *pgxpool.Config) { + if connCfg, ok := cfg.(ConfigWithMaxConn); ok { + poolCfg.MaxConns = connCfg.GetMaxConnections() + } +} + +func New(cfg MigrationConfig, afterConnect ...AfterConnect) *pgxpool.Pool { + + if len(afterConnect) == 0 { + afterConnect = []AfterConnect{AddUUIDSupport} + } + + poolConfig, err := pgxpool.ParseConfig(cfg.GetURI()) + if err != nil { + slog.Error("failed to parse postgres config", "error", err) + panic(err) + } + + addMaxConnectionsToCfg(cfg, poolConfig) + if timeoutCfg, ok := cfg.(ConfigWithTimeouts); ok { + poolConfig.ConnConfig.ConnectTimeout = timeoutCfg.GetConnectTimeout() + poolConfig.ConnConfig.RuntimeParams["statement_timeout"] = strconv.FormatInt(timeoutCfg.GetQueryTimeout().Milliseconds(), 10) + } + poolConfig.ConnConfig.RuntimeParams["application_name"] = "mtusi-api" + slog.Info("postgres_pool_config", "max_conns", poolConfig.MaxConns, + "connect_timeout", poolConfig.ConnConfig.ConnectTimeout, + "statement_timeout_ms", poolConfig.ConnConfig.RuntimeParams["statement_timeout"]) + + poolConfig.ConnConfig.Tracer = telemetry.DBTracer{} + + poolConfig.AfterConnect = load(afterConnect...) + + pool, err := pgxpool.NewWithConfig(context.Background(), poolConfig) + if err != nil { + slog.Error("failed to create postgres pool", "error", err) + panic(err) + } + + slog.Info("successfully connected to postgres pool") + return pool +} + +func load(afterConnects ...AfterConnect) func(ctx context.Context, conn *pgx.Conn) error { + return func(ctx context.Context, conn *pgx.Conn) error { + for _, afterConnect := range afterConnects { + if err := afterConnect(ctx, conn); err != nil { + return err + } + } + return nil + } +} diff --git a/service/internal/pkg/postgres/txmanager/tx.go b/service/internal/pkg/postgres/txmanager/tx.go new file mode 100644 index 0000000..c4aa2fe --- /dev/null +++ b/service/internal/pkg/postgres/txmanager/tx.go @@ -0,0 +1,59 @@ +package txmanager + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type TransactionManager struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *TransactionManager { + return &TransactionManager{ + pool: pool, + } +} + +func (tm *TransactionManager) RunInTransaction(ctx context.Context, opts *pgx.TxOptions, fn func(tx pgx.Tx) error) (err error) { + var tx pgx.Tx + + if opts != nil { + tx, err = tm.pool.BeginTx(ctx, *opts) + } else { + tx, err = tm.pool.Begin(ctx) + } + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + + var committed bool + + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("panic in transaction: %v", p) + } + if !committed { + rollbackErr := tx.Rollback(ctx) + if rollbackErr != nil && !errors.Is(rollbackErr, pgx.ErrTxClosed) { + slog.Error("failed to rollback transaction", "error", rollbackErr) + } + } + }() + + if err = fn(tx); err != nil { + return fmt.Errorf("failed to execute transaction: %w", err) + } + + if err = tx.Commit(ctx); err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + + committed = true + return nil +} diff --git a/service/internal/pkg/rest/framework/framework.go b/service/internal/pkg/rest/framework/framework.go new file mode 100644 index 0000000..a50464d --- /dev/null +++ b/service/internal/pkg/rest/framework/framework.go @@ -0,0 +1,257 @@ +package framework + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "mime" + "net/http" + "strconv" + "strings" + + "github.com/go-chi/chi/v5" + v10 "github.com/go-playground/validator/v10" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers" + "gitea.privcloud.tech/mtusi2026/internal/pkg/validator" +) + +func validate(ctx context.Context, v *validator.ValidatorImpl, value any) error { + err := v.ValidateCtx(ctx, value) + if err == nil { + return nil + } + var fieldErrors v10.ValidationErrors + if errors.As(err, &fieldErrors) { + return apperr.Wrap(err, apperr.CodeValidation, "invalid request fields") + } + return apperr.Wrap(err, apperr.CodeInternal, "validation failed") +} + +func Validate(w http.ResponseWriter, r *http.Request, v *validator.ValidatorImpl, value any) bool { + if err := validate(r.Context(), v, value); err != nil { + JSONError(r.Context(), w, err) + return false + } + return true +} + +func JSONResponse(w http.ResponseWriter, status int, body any) { + w.Header().Set(headers.HeaderContentType, "application/json") + w.WriteHeader(status) + + if status == http.StatusNoContent { + return + } + + if err := json.NewEncoder(w).Encode(body); err != nil { + wrapped := apperr.Wrap(err, apperr.CodeInternal, "failed json encode") + slog.Error("response", "error", wrapped) + } +} + +func JSONError(ctx context.Context, w http.ResponseWriter, err error) { + if err == nil { + return + } + + requestId, ok := ctx.Value(headers.HeaderRequestID).(uuid.UUID) + if !ok { + requestId = uuid.New() + } + w.Header().Set(headers.HeaderRequestID, requestId.String()) + + if errors.Is(err, context.DeadlineExceeded) { + err = apperr.Wrap(err, apperr.CodeServiceUnavailable, "request deadline exceeded") + } + appErr := apperr.As(err) + if appErr == nil { + appErr = apperr.As(apperr.New(apperr.CodeInternal, "internal server error")) + } + if appErr.Code().HTTPStatus() == http.StatusUnauthorized && w.Header().Get("WWW-Authenticate") == "" { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + } + + slog.ErrorContext(ctx, "http_error", "error_code", appErr.Code().String(), "status", appErr.Code().HTTPStatus(), "request_id", requestId.String()) + + var code contracts.ErrorErrorCode + switch appErr.Code() { + case apperr.CodeBadRequest, apperr.CodeValidation: + code = contracts.InvalidRequest + case apperr.CodeUnauthorized: + code = contracts.Unauthorized + case apperr.CodeNotFound: + code = contracts.NotFound + case apperr.CodeAlreadyExists, apperr.CodeConflict: + code = contracts.Conflict + case apperr.CodePayloadTooLarge: + code = contracts.PayloadTooLarge + case apperr.CodeTooManyRequests: + code = contracts.RateLimited + case apperr.CodeServiceUnavailable, apperr.CodeUnavailable: + code = contracts.Unavailable + case apperr.CodeUnsupportedMediaType: + code = contracts.UnsupportedMediaType + default: + code = contracts.InternalError + } + JSONResponse(w, appErr.Code().HTTPStatus(), contracts.Error{ + Error: struct { + Code contracts.ErrorErrorCode `json:"code"` + Message string `json:"message"` + RequestId string `json:"request_id"` + }{ + Code: code, + Message: appErr.Message(), + RequestId: requestId.String(), + }, + }) +} + +func Decode(w http.ResponseWriter, r *http.Request, object any) bool { + mediaType, params, err := mime.ParseMediaType(r.Header.Get(headers.HeaderContentType)) + if err != nil || mediaType != "application/json" || + (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) { + JSONError(r.Context(), w, apperr.New(apperr.CodeUnsupportedMediaType, "expected application/json")) + return false + } + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + if err := decoder.Decode(object); err != nil { + JSONError(r.Context(), w, apperr.Wrap(err, apperr.CodeBadRequest, "failed decode")) + return false + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "expected one JSON object")) + return false + } + return true +} + +func PathParam(r *http.Request, key string) string { + return chi.URLParam(r, key) +} + +func QueryParam(r *http.Request, key string) string { + return r.URL.Query().Get(key) +} + +func HeaderParam(r *http.Request, key string) string { + return r.Header.Get(key) +} + +func ParseIDFromPathParam(ctx context.Context, w http.ResponseWriter, r *http.Request, key string) (uuid.UUID, error) { + rawID := PathParam(r, key) + id, err := uuid.Parse(rawID) + if err != nil { + JSONError(ctx, w, apperr.New(apperr.CodeValidation, fmt.Sprintf("provided invalid %s", key))) + return uuid.Nil, err + } + return id, nil +} + +func ParseIDFromQueryParam(ctx context.Context, w http.ResponseWriter, r *http.Request, key string) (uuid.UUID, error) { + rawID := QueryParam(r, key) + id, err := uuid.Parse(rawID) + if err != nil { + JSONError(ctx, w, apperr.New(apperr.CodeValidation, fmt.Sprintf("provided invalid %s", key))) + return uuid.Nil, err + } + return id, nil +} + +func ParseIDFromHeaderParam(ctx context.Context, w http.ResponseWriter, r *http.Request, key string) (uuid.UUID, error) { + rawID := HeaderParam(r, key) + id, err := uuid.Parse(rawID) + if err != nil { + JSONError(ctx, w, apperr.New(apperr.CodeUnauthorized, "missing or invalid user ID")) + return uuid.Nil, err + } + return id, nil +} + +func Header(r *http.Request, header string) (string, bool) { + return r.Header.Get(header), r.Header.Get(header) != "" +} + +func Token(r *http.Request) (string, bool) { + foundHeader := r.Header.Get(headers.HeaderAuthorization) + if foundHeader == "" { + return "", false + } + + token, found := strings.CutPrefix(foundHeader, "Bearer ") + if !found { + return "", false + } + + return token, true +} + +type sessionContextKey struct{} + +func WithVerifiedSession(r *http.Request, data *dto.UserDataInToken) *http.Request { + return r.WithContext(context.WithValue(r.Context(), sessionContextKey{}, *data)) +} + +func ParseToken(r *http.Request, provider jwtprovider.Provider) (*dto.UserDataInToken, error) { + if data, ok := r.Context().Value(sessionContextKey{}).(dto.UserDataInToken); ok { + return &data, nil + } + foundHeader := r.Header.Get(headers.HeaderAuthorization) + if foundHeader == "" { + return nil, apperr.New(apperr.CodeUnauthorized, "missing authorization header") + } + + token, found := strings.CutPrefix(foundHeader, "Bearer ") + if !found { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid authorization header") + } + + userData, err := provider.ParseVerified(&dto.Token{Payload: token}) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeUnauthorized, "invalid token") + } + + return userData, nil +} + +func PageParams(w http.ResponseWriter, r *http.Request) (*int, *uuid.UUID, bool) { + query := r.URL.Query() + var limit *int + if values, exists := query["limit"]; exists { + if len(values) != 1 { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid limit")) + return nil, nil, false + } + value, err := strconv.Atoi(values[0]) + if err != nil || value < 1 || value > 100 { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid limit")) + return nil, nil, false + } + limit = &value + } + var afterID *uuid.UUID + if values, exists := query["cursor"]; exists { + if len(values) != 1 { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid cursor")) + return nil, nil, false + } + id, err := uuid.Parse(values[0]) + if err != nil || id == uuid.Nil { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid cursor")) + return nil, nil, false + } + afterID = &id + } + return limit, afterID, true +} diff --git a/service/internal/pkg/rest/headers/headers.go b/service/internal/pkg/rest/headers/headers.go new file mode 100644 index 0000000..c92a8de --- /dev/null +++ b/service/internal/pkg/rest/headers/headers.go @@ -0,0 +1,8 @@ +package headers + +const ( + HeaderContentType = "Content-Type" + HeaderRequestID = "X-Request-Id" + HeaderUserID = "X-User-Id" + HeaderAuthorization = "Authorization" +) diff --git a/service/internal/pkg/rest/middleware/auth.go b/service/internal/pkg/rest/middleware/auth.go new file mode 100644 index 0000000..a39c053 --- /dev/null +++ b/service/internal/pkg/rest/middleware/auth.go @@ -0,0 +1,75 @@ +package middleware + +import ( + "context" + "net/http" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +var _ AuthMiddleware = (*authMiddleware)(nil) + +type AuthMiddleware interface { + WithPrivateAccess(next http.Handler) http.Handler + WithAdminAccess(next http.Handler) http.Handler +} + +type SessionVerifier interface { + VerifySession(ctx context.Context, token dto.Token) (*dto.UserDataInToken, error) +} + +type authMiddleware struct { + verifier SessionVerifier +} + +func NewAuthMiddleware(verifier SessionVerifier) AuthMiddleware { + return &authMiddleware{verifier: verifier} +} + +func (a *authMiddleware) WithPrivateAccess(next http.Handler) http.Handler { + return a.privateAccessCheckCallback(next) +} + +func (a *authMiddleware) WithAdminAccess(next http.Handler) http.Handler { + return a.operatorAccessCheckCallback(role.Admin, next) +} + +func (a *authMiddleware) privateAccessCheckCallback(next http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + data, err := a.verifySession(r) + if err != nil { + framework.JSONError(r.Context(), w, err) + return + } + + next.ServeHTTP(w, framework.WithVerifiedSession(r, data)) + } +} + +func (a *authMiddleware) operatorAccessCheckCallback(expectedRole role.Role, next http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + u, err := a.verifySession(r) + if err != nil { + framework.JSONError(r.Context(), w, err) + return + } + + if u.Role != role.Admin && u.Role != expectedRole { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeForbidden, "insufficient permissions")) + return + } + + next.ServeHTTP(w, framework.WithVerifiedSession(r, u)) + } +} + +func (a *authMiddleware) verifySession(r *http.Request) (*dto.UserDataInToken, error) { + token, found := framework.Token(r) + if !found || token == "" { + return nil, apperr.New(apperr.CodeUnauthorized, "missing or invalid bearer token") + } + return a.verifier.VerifySession(r.Context(), dto.Token{Payload: token}) +} diff --git a/service/internal/pkg/rest/middleware/auth_test.go b/service/internal/pkg/rest/middleware/auth_test.go new file mode 100644 index 0000000..b62e5d8 --- /dev/null +++ b/service/internal/pkg/rest/middleware/auth_test.go @@ -0,0 +1,73 @@ +package middleware + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "github.com/google/uuid" +) + +type sessionVerifier struct { + calls int + data dto.UserDataInToken + err error +} + +func (v *sessionVerifier) VerifySession(context.Context, dto.Token) (*dto.UserDataInToken, error) { + v.calls++ + return &v.data, v.err +} + +func TestVerifiedSessionIsRequestScoped(t *testing.T) { + verifier := &sessionVerifier{data: dto.UserDataInToken{UserID: uuid.New()}} + middleware := NewAuthMiddleware(verifier) + handled := 0 + handler := middleware.WithPrivateAccess(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // A nil provider is intentional: the handler must use the verified request identity. + data, err := framework.ParseToken(r, nil) + if err != nil { + t.Fatal(err) + } + if data.UserID != verifier.data.UserID { + t.Fatal("verified identity changed") + } + handled++ + w.WriteHeader(http.StatusNoContent) + })) + for range 2 { + request := httptest.NewRequest("GET", "/private", nil) + request.Header.Set("Authorization", "Bearer synthetic-token") + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + if recorder.Code != 204 { + t.Fatal(recorder.Code) + } + } + if verifier.calls != 2 || handled != 2 { + t.Fatal("session verification must run for every request") + } + verifier.err = apperr.New(apperr.CodeUnauthorized, "revoked session") + request := httptest.NewRequest("GET", "/private", nil) + request.Header.Set("Authorization", "Bearer synthetic-token") + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + if recorder.Code != 401 || handled != 2 { + t.Fatal("revocation was bypassed by cached request identity") + } +} + +func TestMissingTokenDoesNotCallSessionStore(t *testing.T) { + verifier := &sessionVerifier{err: errors.New("must not be called")} + handler := NewAuthMiddleware(verifier).WithPrivateAccess(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("handler called without a token") })) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest("GET", "/private", nil)) + if recorder.Code != 401 || verifier.calls != 0 { + t.Fatal("missing token was accepted") + } +} diff --git a/service/internal/pkg/rest/middleware/instance.go b/service/internal/pkg/rest/middleware/instance.go new file mode 100644 index 0000000..6447705 --- /dev/null +++ b/service/internal/pkg/rest/middleware/instance.go @@ -0,0 +1,12 @@ +package middleware + +import "net/http" + +func Instance(id string) Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", id) + next.ServeHTTP(w, r) + }) + } +} diff --git a/service/internal/pkg/rest/middleware/logging.go b/service/internal/pkg/rest/middleware/logging.go new file mode 100644 index 0000000..8ffda7a --- /dev/null +++ b/service/internal/pkg/rest/middleware/logging.go @@ -0,0 +1,34 @@ +package middleware + +import ( + "log/slog" + "net/http" + "time" + + "github.com/go-chi/chi/v5/middleware" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers" +) + +func LoggingMiddleware() Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + started := time.Now() + wrapped := middleware.NewWrapResponseWriter(w, r.ProtoMajor) + next.ServeHTTP(wrapped, r) + + status := wrapped.Status() + if status == 0 { + status = http.StatusOK + } + slog.InfoContext(r.Context(), "http request", + "method", r.Method, + "path", r.URL.Path, + "status", status, + "duration", time.Since(started), + "bytes", wrapped.BytesWritten(), + "request_id", w.Header().Get(headers.HeaderRequestID), + ) + }) + } +} diff --git a/service/internal/pkg/rest/middleware/middleware.go b/service/internal/pkg/rest/middleware/middleware.go new file mode 100644 index 0000000..f54cea9 --- /dev/null +++ b/service/internal/pkg/rest/middleware/middleware.go @@ -0,0 +1,5 @@ +package middleware + +import "net/http" + +type Middleware = func(http.Handler) http.Handler diff --git a/service/internal/pkg/rest/middleware/request-id.go b/service/internal/pkg/rest/middleware/request-id.go new file mode 100644 index 0000000..eed3549 --- /dev/null +++ b/service/internal/pkg/rest/middleware/request-id.go @@ -0,0 +1,22 @@ +package middleware + +import ( + "context" + "net/http" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers" +) + +func RequestIDMiddleware() Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + requestID := uuid.New() + w.Header().Set(headers.HeaderRequestID, requestID.String()) + ctx = context.WithValue(ctx, headers.HeaderRequestID, requestID) + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} diff --git a/service/internal/pkg/rest/middleware/timeout.go b/service/internal/pkg/rest/middleware/timeout.go new file mode 100644 index 0000000..7542147 --- /dev/null +++ b/service/internal/pkg/rest/middleware/timeout.go @@ -0,0 +1,23 @@ +package middleware + +import ( + "context" + "net/http" + "strings" + "time" +) + +// RequestTimeout bounds database work and waiting for a pooled connection. +func RequestTimeout(timeout time.Duration, attachmentTimeout ...time.Duration) Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestTimeout := timeout + if len(attachmentTimeout) > 0 && strings.Contains(r.URL.Path, "/attachments") { + requestTimeout = attachmentTimeout[0] + } + ctx, cancel := context.WithTimeout(r.Context(), requestTimeout) + defer cancel() + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} diff --git a/service/internal/pkg/rest/router/router.go b/service/internal/pkg/rest/router/router.go new file mode 100644 index 0000000..6c67300 --- /dev/null +++ b/service/internal/pkg/rest/router/router.go @@ -0,0 +1,135 @@ +package router + +import ( + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "net/http" + + "github.com/go-chi/chi/v5" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware" +) + +type Router interface { + Private(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) + Public(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) + Admin(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) + ServeHTTP(w http.ResponseWriter, r *http.Request) + With(mw ...func(http.Handler) http.Handler) Router + Use(mw ...func(http.Handler) http.Handler) Router + Handle(pattern string, handler http.Handler) + Get(pattern string, handler http.HandlerFunc) + Head(pattern string, handler http.HandlerFunc) + Options(pattern string, handler http.HandlerFunc) + Post(pattern string, handler http.HandlerFunc) + Put(pattern string, handler http.HandlerFunc) + Patch(pattern string, handler http.HandlerFunc) + Delete(pattern string, handler http.HandlerFunc) + Connect(pattern string, handler http.HandlerFunc) + Trace(pattern string, handler http.HandlerFunc) +} + +type router struct { + r chi.Router + jwtMw middleware.AuthMiddleware +} + +func New(jwtMw middleware.AuthMiddleware, mw ...middleware.Middleware) Router { + r := chi.NewRouter() + r.Use(mw...) + return &router{ + r: r, + jwtMw: jwtMw, + } +} + +func (c *router) Group(pattern string) Router { + sub := &router{ + r: chi.NewRouter(), + jwtMw: c.jwtMw, + } + c.r.Mount(pattern, sub.r) + return sub +} + +func (c *router) Public(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) { + telemetry.RegisterRoute(method, pattern) + next := c.applyMiddlewares(handler, mw...) + c.r.MethodFunc(method, pattern, next.ServeHTTP) +} + +func (c *router) Private(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) { + telemetry.RegisterRoute(method, pattern) + next := c.applyMiddlewares(handler, mw...) + c.r.MethodFunc(method, pattern, (c.jwtMw.WithPrivateAccess(next)).ServeHTTP) +} + +func (c *router) Admin(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) { + telemetry.RegisterRoute(method, pattern) + next := c.applyMiddlewares(handler, mw...) + c.r.MethodFunc(method, pattern, (c.jwtMw.WithAdminAccess(next)).ServeHTTP) +} + +func (c *router) ServeHTTP(w http.ResponseWriter, r *http.Request) { + c.r.ServeHTTP(w, r) +} + +func (c *router) With(mw ...func(http.Handler) http.Handler) Router { + sub := &router{ + r: c.r.With(mw...), + jwtMw: c.jwtMw, + } + return sub +} + +func (c *router) Use(mw ...func(http.Handler) http.Handler) Router { + c.r.Use(mw...) + return c +} + +func (c *router) Handle(pattern string, handler http.Handler) { + c.r.Handle(pattern, handler) +} + +func (c *router) Get(pattern string, handler http.HandlerFunc) { + c.r.Get(pattern, handler) +} + +func (c *router) Head(pattern string, handler http.HandlerFunc) { + c.r.Head(pattern, handler) +} + +func (c *router) Options(pattern string, handler http.HandlerFunc) { + c.r.Options(pattern, handler) +} + +func (c *router) Post(pattern string, handler http.HandlerFunc) { + c.r.Post(pattern, handler) +} + +func (c *router) Put(pattern string, handler http.HandlerFunc) { + c.r.Put(pattern, handler) +} + +func (c *router) Patch(pattern string, handler http.HandlerFunc) { + c.r.Patch(pattern, handler) +} + +func (c *router) Delete(pattern string, handler http.HandlerFunc) { + c.r.Delete(pattern, handler) +} + +func (c *router) Connect(pattern string, handler http.HandlerFunc) { + c.r.Connect(pattern, handler) +} + +func (c *router) Trace(pattern string, handler http.HandlerFunc) { + c.r.Trace(pattern, handler) +} + +func (c *router) applyMiddlewares(handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) http.Handler { + next := http.Handler(handler) + for _, mwFunc := range mw { + next = mwFunc(next) + } + return next +} diff --git a/service/internal/pkg/telemetry/http.go b/service/internal/pkg/telemetry/http.go new file mode 100644 index 0000000..d0399be --- /dev/null +++ b/service/internal/pkg/telemetry/http.go @@ -0,0 +1,56 @@ +package telemetry + +import ( + "context" + "github.com/go-chi/chi/v5" + mw "github.com/go-chi/chi/v5/middleware" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "log/slog" + "net/http" + "strconv" + "time" +) + +func HTTP(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, sp := Start(Extract(r.Context(), r.Header.Get("traceparent"), r.Header.Get("tracestate")), "HTTP", trace.SpanKindServer) + start := time.Now() + ww := mw.NewWrapResponseWriter(w, r.ProtoMajor) + defer func() { + route := chi.RouteContext(ctx).RoutePattern() + if route == "" { + route = "unmatched" + } + method := r.Method + switch method { + case "GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS": + default: + method = "OTHER" + } + status := ww.Status() + if status == 0 { + status = 200 + } + sp.SetName(method + " " + route) + sp.SetAttributes(attribute.String("http.route", route), attribute.String("http.request.method", method), attribute.Int("http.response.status_code", status)) + if status >= 500 { + sp.SetStatus(codes.Error, "server_error") + } + a := []attribute.KeyValue{attribute.String("method", method), attribute.String("route", route), attribute.String("status_class", strconv.Itoa(status/100)+"xx")} + Count(ctx, "messenger.http.requests", 1, a...) + Duration(ctx, "messenger.http.duration", time.Since(start), a[:2]...) + slog.InfoContext(ctx, "http_request", "method", method, "route", route, "status", status, "request_id", w.Header().Get("X-Request-Id")) + sp.End() + }() + next.ServeHTTP(ww, r.WithContext(ctx)) + }) +} + +// RegisterRoute publishes zero counters before any outcome, preserving the first spike. +func RegisterRoute(method, route string) { + for _, class := range []string{"1xx", "2xx", "3xx", "4xx", "5xx"} { + Count(context.Background(), "messenger.http.requests", 0, attribute.String("method", method), attribute.String("route", route), attribute.String("status_class", class)) + } +} diff --git a/service/internal/pkg/telemetry/observation.go b/service/internal/pkg/telemetry/observation.go new file mode 100644 index 0000000..22063f0 --- /dev/null +++ b/service/internal/pkg/telemetry/observation.go @@ -0,0 +1,57 @@ +package telemetry + +import ( + "context" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "sync" + "sync/atomic" + "time" +) + +var QueueDepth atomic.Int64 +var observationMu sync.RWMutex + +type observationSnapshot struct { + Queued, Outbox, Good, Bad int64 + Locked int64 + Oldest float64 +} + +var snapshot observationSnapshot +var observed time.Time + +func PublishObservation(queued, outbox, good, bad int64, oldest float64, locked int64) { + s := observationSnapshot{Queued: queued, Outbox: outbox, Good: good, Bad: bad, Oldest: oldest, Locked: locked} + observationMu.Lock() + snapshot = s + observed = time.Now() + observationMu.Unlock() +} +func RegisterObservation() { + locks, _ := meter.Int64ObservableGauge("messenger.db.locks.waiting") + pending, _ := meter.Int64ObservableGauge("messenger.jobs.pending") + outbox, _ := meter.Int64ObservableGauge("messenger.outbox.pending") + queue, _ := meter.Int64ObservableGauge("messenger.queue.depth") + age, _ := meter.Float64ObservableGauge("messenger.queue.oldest.age", metric.WithUnit("s")) + fresh, _ := meter.Float64ObservableGauge("messenger.observation.timestamp") + slo, _ := meter.Int64ObservableCounter("messenger.ready.slo") + _, _ = meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error { + observationMu.RLock() + s, t := snapshot, observed + observationMu.RUnlock() + if t.IsZero() { + return nil + } + o.ObserveInt64(locks, s.Locked) + o.ObserveInt64(pending, s.Queued) + o.ObserveInt64(outbox, s.Outbox) + o.ObserveInt64(queue, QueueDepth.Load(), metric.WithAttributes(attribute.String("queue", "attachments.v1"))) + o.ObserveFloat64(age, s.Oldest, metric.WithAttributes(attribute.String("queue", "attachments.v1"))) + o.ObserveFloat64(fresh, float64(t.Unix())) + o.ObserveInt64(slo, s.Good, metric.WithAttributes(attribute.String("outcome", "good"))) + o.ObserveInt64(slo, s.Bad, metric.WithAttributes(attribute.String("outcome", "bad"))) + return nil + }, locks, pending, outbox, queue, age, fresh, slo) +} diff --git a/service/internal/pkg/telemetry/process.go b/service/internal/pkg/telemetry/process.go new file mode 100644 index 0000000..8f8b495 --- /dev/null +++ b/service/internal/pkg/telemetry/process.go @@ -0,0 +1,76 @@ +package telemetry + +import ( + "context" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/shirou/gopsutil/v4/process" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + "os" + "strings" + "sync/atomic" + "time" +) + +var waiting atomic.Int64 + +func ObservePool(pool *pgxpool.Pool, capacity int) { + names := []string{"messenger.db.pool.used", "messenger.db.pool.capacity", "messenger.db.pool.waiting", "messenger.worker.busy", "messenger.worker.capacity", "messenger.process.memory", "messenger.process.cpu", "messenger.telemetry.export_errors"} + instruments := make([]metric.Float64ObservableGauge, len(names)) + for i, n := range names { + instruments[i], _ = meter.Float64ObservableGauge(n) + } + p, _ := process.NewProcess(int32(os.Getpid())) + obs := make([]metric.Observable, len(instruments)) + for i, g := range instruments { + obs[i] = g + } + _, _ = meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error { + st := pool.Stat() + v := []float64{float64(st.AcquiredConns()), float64(st.MaxConns()), float64(waiting.Load()), float64(Busy.Load()), float64(capacity), 0, 0, float64(ExportErrors.Load())} + if p != nil { + if m, e := p.MemoryInfoWithContext(ctx); e == nil { + v[5] = float64(m.RSS) + } + if t, e := p.TimesWithContext(ctx); e == nil { + v[6] = t.User + t.System + } + } + for i, g := range instruments { + o.ObserveFloat64(g, v[i]) + } + return nil + }, obs...) +} + +type DBTracer struct{} +type acquireKey struct{} + +func (DBTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, d pgx.TraceQueryStartData) context.Context { + op := "OTHER" + f := strings.Fields(d.SQL) + if len(f) > 0 { + switch strings.ToUpper(f[0]) { + case "SELECT", "INSERT", "UPDATE", "DELETE", "BEGIN", "COMMIT", "ROLLBACK": + op = strings.ToUpper(f[0]) + } + } + ctx, sp := Start(ctx, "postgresql "+op, trace.SpanKindClient) + sp.SetAttributes(attribute.String("db.system.name", "postgresql"), attribute.String("db.operation.name", op)) + return ctx +} +func (DBTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, d pgx.TraceQueryEndData) { + End(trace.SpanFromContext(ctx), d.Err) +} +func (DBTracer) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context { + waiting.Add(1) + return context.WithValue(ctx, acquireKey{}, time.Now()) +} +func (DBTracer) TraceAcquireEnd(ctx context.Context, _ *pgxpool.Pool, d pgxpool.TraceAcquireEndData) { + waiting.Add(-1) + if t, ok := ctx.Value(acquireKey{}).(time.Time); ok { + Duration(ctx, "messenger.db.acquire.duration", time.Since(t)) + } +} diff --git a/service/internal/pkg/telemetry/telemetry.go b/service/internal/pkg/telemetry/telemetry.go new file mode 100644 index 0000000..6cb7d37 --- /dev/null +++ b/service/internal/pkg/telemetry/telemetry.go @@ -0,0 +1,136 @@ +// Package telemetry owns signal export and safe, low-cardinality instrumentation. +package telemetry + +import ( + "context" + "log/slog" + "os" + "strings" + "sync/atomic" + "time" + + "github.com/google/uuid" + "go.opentelemetry.io/contrib/bridges/otelslog" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/propagation" + sdklog "go.opentelemetry.io/otel/sdk/log" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +var InstanceID = uuid.NewString() +var Busy atomic.Int64 +var meter = otel.Meter("messenger") +var tracer = otel.Tracer("messenger") + +// A fanout retains local JSON and exports the same structured event over OTLP. +type handler struct{ local, remote slog.Handler } + +func (h handler) Enabled(c context.Context, l slog.Level) bool { return h.local.Enabled(c, l) } +func (h handler) Handle(c context.Context, r slog.Record) error { + // Errors can contain SQL values, credentials or invalid request content. + safe := slog.NewRecord(r.Time, r.Level, r.Message, r.PC) + r.Attrs(func(a slog.Attr) bool { + key := strings.ToLower(a.Key) + switch key { + case "authorization", "password", "token", "session_token", "body", "filename", "url": + return true + case "error": + a = slog.String("error.type", "operation_failed") + } + safe.AddAttrs(a) + return true + }) + r = safe + sc := trace.SpanContextFromContext(c) + r.AddAttrs(slog.String("event", r.Message)) + if sc.IsValid() { + r.AddAttrs(slog.String("trace_id", sc.TraceID().String()), slog.String("span_id", sc.SpanID().String())) + } + _ = h.remote.Handle(c, r) + return h.local.Handle(c, r) +} +func (h handler) WithAttrs(a []slog.Attr) slog.Handler { + return handler{h.local.WithAttrs(a), h.remote.WithAttrs(a)} +} +func (h handler) WithGroup(g string) slog.Handler { + return handler{h.local.WithGroup(g), h.remote.WithGroup(g)} +} + +func Init(ctx context.Context, name string) (func(), error) { + otel.SetTextMapPropagator(propagation.TraceContext{}) + if os.Getenv("OTEL_SDK_DISABLED") == "true" { + return func() {}, nil + } + res := resource.NewWithAttributes("", attribute.String("service.name", name), attribute.String("service.version", "lab6"), attribute.String("service.instance.id", InstanceID), attribute.String("deployment.environment.name", "lab")) + te, e := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(time.Second)) + if e != nil { + return nil, e + } + tp := sdktrace.NewTracerProvider(sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.AlwaysSample())), sdktrace.WithBatcher(te, sdktrace.WithMaxQueueSize(1024), sdktrace.WithMaxExportBatchSize(128), sdktrace.WithBatchTimeout(time.Second), sdktrace.WithExportTimeout(time.Second))) + me, e := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(time.Second)) + if e != nil { + return nil, e + } + mp := sdkmetric.NewMeterProvider(sdkmetric.WithResource(res), sdkmetric.WithReader(sdkmetric.NewPeriodicReader(me, sdkmetric.WithInterval(5*time.Second), sdkmetric.WithTimeout(time.Second)))) + le, e := otlploghttp.New(ctx, otlploghttp.WithTimeout(time.Second)) + if e != nil { + return nil, e + } + lp := sdklog.NewLoggerProvider(sdklog.WithResource(res), sdklog.WithProcessor(sdklog.NewBatchProcessor(le, sdklog.WithMaxQueueSize(1024), sdklog.WithExportMaxBatchSize(128), sdklog.WithExportInterval(time.Second), sdklog.WithExportTimeout(time.Second)))) + otel.SetTracerProvider(tp) + otel.SetMeterProvider(mp) + if name == "messenger-worker" { + for _, outcome := range []string{"ready", "retry", "failed", "duplicate"} { + Count(ctx, "messenger.jobs", 0, attribute.String("job_type", "attachment.process.v1"), attribute.String("outcome", outcome)) + } + } + local := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}).WithAttrs([]slog.Attr{slog.String("service.name", name), slog.String("service.instance.id", InstanceID)}) + slog.SetDefault(slog.New(handler{local, otelslog.NewHandler("messenger", otelslog.WithLoggerProvider(lp))})) + // Export failures are summarized safely and never include payloads or endpoints. + otel.SetErrorHandler(otel.ErrorHandlerFunc(func(error) { ExportErrors.Add(1) })) + return func() { + c, cancel := context.WithTimeout(context.Background(), 4*time.Second) + defer cancel() + _ = mp.Shutdown(c) + _ = lp.Shutdown(c) + _ = tp.Shutdown(c) + }, nil +} + +var ExportErrors atomic.Int64 + +func Start(ctx context.Context, name string, kind trace.SpanKind) (context.Context, trace.Span) { + return tracer.Start(ctx, name, trace.WithSpanKind(kind)) +} +func End(span trace.Span, err error) { + if err != nil { + span.SetStatus(codes.Error, "operation_failed") + span.SetAttributes(attribute.String("error.type", "dependency_or_processing")) + } + span.End() +} +func Inject(ctx context.Context) (string, string) { + c := propagation.MapCarrier{} + otel.GetTextMapPropagator().Inject(ctx, c) + return c.Get("traceparent"), c.Get("tracestate") +} +func Extract(ctx context.Context, p, s string) context.Context { + return otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier{"traceparent": p, "tracestate": s}) +} +func Count(ctx context.Context, name string, n int64, a ...attribute.KeyValue) { + c, _ := meter.Int64Counter(name) + c.Add(ctx, n, metric.WithAttributes(a...)) +} +func Duration(ctx context.Context, name string, d time.Duration, a ...attribute.KeyValue) { + h, _ := meter.Float64Histogram(name, metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(.005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 20, 30, 60, 120, 300)) + h.Record(ctx, d.Seconds(), metric.WithAttributes(a...)) +} diff --git a/service/internal/pkg/types/authprovider/provider.go b/service/internal/pkg/types/authprovider/provider.go new file mode 100644 index 0000000..b05721c --- /dev/null +++ b/service/internal/pkg/types/authprovider/provider.go @@ -0,0 +1,82 @@ +package authprovider + +import ( + "encoding/json" + "strings" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types" +) + +type Type string + +const ( + TypeUndefined Type = "undefined" + TypePassword Type = "password" + TypeEmail Type = "email" + TypePhone Type = "phone" + TypeGoogle Type = "google" + TypeTelegram Type = "telegram" +) + +var allowedTypes = map[Type]struct{}{ + TypePassword: {}, + TypeEmail: {}, + TypePhone: {}, + TypeGoogle: {}, + TypeTelegram: {}, +} + +func ParseType(value string) Type { + t := Type(strings.ToLower(value)) + if !t.IsValid() { + return TypeUndefined + } + return t +} + +func ParseAndValidateType(value string) (Type, error) { + parsed := ParseType(value) + if _, ok := allowedTypes[parsed]; ok { + return parsed, nil + } + return TypeUndefined, types.ErrUnsupportedOTPStatus +} + +func (t Type) String() string { + return string(t) +} + +func (t Type) Validate() error { + if _, ok := allowedTypes[t]; ok { + return nil + } + return types.ErrUnsupportedAuthProviderType +} + +func (t Type) IsValid() bool { + _, ok := allowedTypes[t] + return ok +} + +func (t Type) MarshalJSON() ([]byte, error) { + if err := t.Validate(); err != nil { + return nil, err + } + return json.Marshal(t.String()) +} + +func (t *Type) UnmarshalJSON(data []byte) error { + if t == nil { + return types.ErrUnmarshalAuthProviderTypeIntoNilPointer + } + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + parsed := ParseType(value) + if err := parsed.Validate(); err != nil { + return err + } + *t = parsed + return nil +} diff --git a/service/internal/pkg/types/environment/environment.go b/service/internal/pkg/types/environment/environment.go new file mode 100644 index 0000000..c1bbd5f --- /dev/null +++ b/service/internal/pkg/types/environment/environment.go @@ -0,0 +1,77 @@ +package environment + +import ( + "encoding/json" + "strings" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types" +) + +type ApplicationEnvironment string + +const ( + ApplicationEnvironmentUndefined ApplicationEnvironment = "undefined" + ApplicationEnvironmentDev ApplicationEnvironment = "dev" + ApplicationEnvironmentProd ApplicationEnvironment = "prod" +) + +var allowedEnvironments = map[ApplicationEnvironment]struct{}{ + ApplicationEnvironmentUndefined: {}, + ApplicationEnvironmentDev: {}, + ApplicationEnvironmentProd: {}, +} + +func ParseEnvironment(value string) ApplicationEnvironment { + s := ApplicationEnvironment(strings.ToLower(value)) + if !s.IsValid() { + return ApplicationEnvironmentUndefined + } + return s +} + +func ParseAndValidateEnvironment(status string) (ApplicationEnvironment, error) { + parsed := ParseEnvironment(status) + if _, ok := allowedEnvironments[parsed]; ok { + return parsed, nil + } + return ApplicationEnvironmentUndefined, types.ErrUnsupportedApplicationEnvironment +} + +func (t ApplicationEnvironment) String() string { + return string(t) +} + +func (t ApplicationEnvironment) Validate() error { + if _, ok := allowedEnvironments[t]; ok { + return nil + } + return types.ErrUnsupportedApplicationEnvironment +} + +func (t ApplicationEnvironment) IsValid() bool { + _, ok := allowedEnvironments[t] + return ok +} + +func (t ApplicationEnvironment) MarshalJSON() ([]byte, error) { + if err := t.Validate(); err != nil { + return nil, err + } + return json.Marshal(t.String()) +} + +func (t *ApplicationEnvironment) UnmarshalJSON(data []byte) error { + if t == nil { + return types.ErrUnmarshalApplicationEnvironmentIntoNilPointer + } + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + parsed := ParseEnvironment(value) + if err := parsed.Validate(); err != nil { + return err + } + *t = parsed + return nil +} diff --git a/service/internal/pkg/types/error.go b/service/internal/pkg/types/error.go new file mode 100644 index 0000000..5f52357 --- /dev/null +++ b/service/internal/pkg/types/error.go @@ -0,0 +1,25 @@ +package types + +import "errors" + +var ( + ErrUnsupportedRole = errors.New("unsupported role") + ErrUnsupportedOTPType = errors.New("unsupported OTP type") + ErrUnsupportedAuthProviderType = errors.New("unsupported auth provider type") + ErrUnsupportedContactType = errors.New("unsupported contact type") + ErrUnsupportedOTPStatus = errors.New("unsupported OTP status") + ErrUnsupportedOTPResult = errors.New("unsupported OTP result") + ErrUnsupportedFormStatus = errors.New("unsupported form status") + ErrUnsupportedAuthIdentityStatus = errors.New("unsupported auth identity status") + ErrUnsupportedApplicationEnvironment = errors.New("unsupported application environment") + + ErrUnmarshalRoleIntoNilPointer = errors.New("unmarshal role into nil pointer") + ErrUnmarshalOTPTypeIntoNilPointer = errors.New("unmarshal OTP type into nil pointer") + ErrUnmarshalAuthProviderTypeIntoNilPointer = errors.New("unmarshal auth provider into nil pointer") + ErrUnmarshalContactTypeIntoNilPointer = errors.New("unmarshal contact into nil pointer") + ErrUnmarshalOTPStatusIntoNilPointer = errors.New("unmarshal OTP status into nil pointer") + ErrUnmarshalOTPResultIntoNilPointer = errors.New("unmarshal OTP result into nil pointer") + ErrUnmarshalFormStatusIntoNilPointer = errors.New("unmarshal form status into nil pointer") + ErrUnmarshalAuthIdentityStatusIntoNilPointer = errors.New("unmarshal auth identity status into nil pointer") + ErrUnmarshalApplicationEnvironmentIntoNilPointer = errors.New("unmarshal application environment into nil pointer") +) diff --git a/service/internal/pkg/types/role/role.go b/service/internal/pkg/types/role/role.go new file mode 100644 index 0000000..6f79d78 --- /dev/null +++ b/service/internal/pkg/types/role/role.go @@ -0,0 +1,105 @@ +package role + +import ( + "encoding/json" + "strings" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types" +) + +type Role string + +const ( + Undefined Role = "undefined" + User Role = "user" + Doctor Role = "doctor" + Manager Role = "manager" + Admin Role = "admin" +) + +var allowedRoles = map[Role]struct{}{ + User: {}, + Doctor: {}, + Manager: {}, + Admin: {}, +} + +func ParseRole(s string) Role { + r := Role(strings.ToLower(s)) + if !r.IsValid() { + return Undefined + } + return r +} + +func ParseAndValidateRole(s string) (Role, error) { + parsed := ParseRole(s) + if _, ok := allowedRoles[parsed]; ok { + return parsed, nil + } + return Undefined, types.ErrUnsupportedRole +} + +func (t Role) String() string { + return string(t) +} + +func (t Role) Validate() error { + if _, ok := allowedRoles[t]; ok { + return nil + } + return types.ErrUnsupportedRole +} + +func (t Role) IsValid() bool { + _, ok := allowedRoles[t] + return ok +} + +func (t Role) MarshalJSON() ([]byte, error) { + if err := t.Validate(); err != nil { + return nil, err + } + return json.Marshal(t.String()) +} + +func (t *Role) UnmarshalJSON(data []byte) error { + if t == nil { + return types.ErrUnmarshalRoleIntoNilPointer + } + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + parsed := ParseRole(value) + if err := parsed.Validate(); err != nil { + return err + } + *t = parsed + return nil +} + +func (t Role) IsAdmin() bool { + return t == Admin +} + +func (t Role) IsManager() bool { + return t == Manager +} + +func (t Role) IsDoctor() bool { + return t == Doctor +} + +func (t Role) IsUser() bool { + return t == User +} + +func (t Role) IsPrivileged() bool { + switch t { + case Doctor, Manager, Admin: + return true + default: + return false + } +} diff --git a/service/internal/pkg/validator/validator.go b/service/internal/pkg/validator/validator.go new file mode 100644 index 0000000..d4ab8aa --- /dev/null +++ b/service/internal/pkg/validator/validator.go @@ -0,0 +1,29 @@ +package validator + +import ( + "context" + + "github.com/go-playground/validator/v10" +) + +type TagKey string + +type ValidatorImpl struct { + validator *validator.Validate +} + +func NewValidatorImpl() *ValidatorImpl { + return &ValidatorImpl{validator: validator.New()} +} + +func (v *ValidatorImpl) Register(tag TagKey, fn validator.Func, callValidationEvenIfNull ...bool) error { + return v.validator.RegisterValidation(string(tag), fn, callValidationEvenIfNull...) +} + +func (v *ValidatorImpl) ValidateCtx(ctx context.Context, value any) error { + return v.validator.StructCtx(ctx, value) +} + +func (v *ValidatorImpl) Validate(value any) error { + return v.validator.Struct(value) +} diff --git a/service/internal/service/attachment/interface.go b/service/internal/service/attachment/interface.go new file mode 100644 index 0000000..0919786 --- /dev/null +++ b/service/internal/service/attachment/interface.go @@ -0,0 +1,55 @@ +package attachment + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "io" + "time" +) + +type TransactionManager interface { + RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error +} +type Outbox interface { + CreateTx(context.Context, pgx.Tx, outbox.CreateMessageReq) (outbox.CreateMessageRes, error) +} +type ObjectStore interface { + Put(context.Context, string, string, io.Reader, int64) error + Get(context.Context, string) (*s3.GetObjectOutput, error) + Head(context.Context, string) (*s3.HeadObjectOutput, error) + Delete(context.Context, string) error + Ping(context.Context) error + List(context.Context, string) (*s3.ListObjectsV2Output, error) +} +type Repository interface { + FailSLO(context.Context, pgx.Tx, uuid.UUID, time.Time) error + AcceptSLO(context.Context, pgx.Tx, uuid.UUID, time.Time, time.Time) error + ReadySLO(context.Context, pgx.Tx, uuid.UUID, time.Time) error + + SaveVariantTx(context.Context, pgx.Tx, uuid.UUID, int, string, string, string, int64) error + + Ping(context.Context) error + Get(context.Context, uuid.UUID) (*dto.AttachmentRecord, error) + GetTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentRecord, error) + InsertTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error + UpdateTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error + JobTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentJob, error) + InsertJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error + SaveJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error + ExpiredJobs(context.Context) ([]uuid.UUID, error) + PublicationDeadTx(context.Context, pgx.Tx, uuid.UUID, int) (bool, error) + FailedOutbox(context.Context) ([]uuid.UUID, error) + ListJobs(context.Context) ([]dto.AttachmentJob, error) + RetiredSources(context.Context, time.Time) ([]*dto.AttachmentRecord, error) + AuditTx(context.Context, pgx.Tx, uuid.UUID, string) error + Lock(context.Context, string) (*pgxpool.Conn, func(), error) + Exists(context.Context, *pgxpool.Conn, string) (bool, error) + Page(context.Context, uuid.UUID) ([]*dto.AttachmentRecord, error) + MarkMissing(context.Context, *pgxpool.Conn, uuid.UUID, bool) (bool, error) + CheckConn(context.Context, *pgxpool.Conn) error +} diff --git a/service/internal/service/attachment/pipeline.go b/service/internal/service/attachment/pipeline.go new file mode 100644 index 0000000..c3d5d24 --- /dev/null +++ b/service/internal/service/attachment/pipeline.go @@ -0,0 +1,555 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/fault" + "gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "time" +) + +var ErrBusy = errors.New("job lease held") + +func validateImage(ctx context.Context, path string) error { return imageprocessor.Validate(ctx, path) } +func (s *Service) Fault(ctx context.Context, point string, id uuid.UUID) { fault.Hit(ctx, point, id) } +func (s *Service) newJob(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error { + now := time.Now().UTC() + j := &dto.AttachmentJob{Message: dto.JobMessage{SchemaVersion: 1, JobID: uuid.New(), Type: "attachment.process.v1", AttachmentID: a.Public.Id, PipelineVersion: a.PipelineVersion, Attempt: 1, CreatedAt: now}, State: "queued", Token: uuid.New(), NextAttemptAt: now, AttemptLimit: s.Jobs.MaxAttempts, UpdatedAt: now} + j.Message.Traceparent, j.Message.Tracestate = telemetry.Inject(ctx) + if e := s.Repo.AcceptSLO(ctx, tx, j.Message.JobID, now, now.Add(30*time.Second)); e != nil { + return e + } + if e := s.Repo.InsertJobTx(ctx, tx, j); e != nil { + return e + } + return s.enqueue(ctx, tx, j) +} +func (s *Service) enqueue(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error { + j.Message.Traceparent, j.Message.Tracestate = telemetry.Inject(ctx) + b, e := json.Marshal(j.Message) + if e != nil { + return e + } + _, e = s.outbox.CreateTx(ctx, tx, outbox.CreateMessageReq{ID: uuid.New(), AggregateID: j.Message.JobID, Transport: outbox.TransportAMQP, Payload: b, CreatedAt: j.NextAttemptAt, MaxAttempts: s.Jobs.MaxAttempts}) + return e +} +func (s *Service) Content(ctx context.Context, user, id uuid.UUID, variant string) (*dto.AttachmentContent, error) { + a, e := s.Get(ctx, user, id) + if e != nil { + return nil, e + } + if a.Public.Status != "ready" { + return nil, apperr.New(apperr.CodeConflict, "attachment not ready") + } + key, mime, size := a.Key, string(a.Public.ContentType), int64(a.Public.SizeBytes) + if variant != "original" { + key = a.VariantKeys[variant] + if key == "" { + return nil, apperr.New(apperr.CodeNotFound, "variant not found") + } + for _, v := range a.Public.Variants { + if string(v.Name) == variant { + mime = string(v.ContentType) + size = int64(v.SizeBytes) + } + } + } + obj, e := s.Store.Get(ctx, key) + if e != nil { + return nil, Unavailable() + } + return &dto.AttachmentContent{Body: obj.Body, ContentType: mime, Size: size}, nil +} +func (s *Service) Process(ctx context.Context, m dto.JobMessage) (processErr error) { + ctx, sp := telemetry.Start(ctx, "attachment.process", trace.SpanKindInternal) + outcome := "unavailable" + started := time.Now() + telemetry.Busy.Add(1) + defer telemetry.Busy.Add(-1) + defer func() { + if outcome == "retry" || outcome == "failed" { + sp.SetStatus(codes.Error, outcome) + } + telemetry.End(sp, processErr) + telemetry.Duration(ctx, "messenger.job.duration", time.Since(started), attribute.String("job_type", m.Type), attribute.String("outcome", outcome)) + }() + + var j *dto.AttachmentJob + var a *dto.AttachmentRecord + claimed := false + err := s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + var e error + j, e = s.Repo.JobTx(ctx, tx, m.JobID) + if errors.Is(e, pgx.ErrNoRows) { + return nil + } + if e != nil { + return e + } + // Broker data is a notification. Trusted metadata and attempt live in PostgreSQL. + if j.Message.AttachmentID != m.AttachmentID || j.Message.PipelineVersion != m.PipelineVersion || j.Message.Attempt != m.Attempt { + return nil + } + a, e = s.Repo.GetTx(ctx, tx, m.AttachmentID) + if e != nil { + return e + } + if a.PipelineVersion != m.PipelineVersion || j.State == "ready" || j.State == "failed" { + return nil + } + if j.State == "processing" { + return ErrBusy + } + if time.Now().Before(j.NextAttemptAt) { + return ErrBusy + } + j.State = "processing" + j.Token = uuid.New() + lease := time.Now().Add(s.Jobs.Lease) + j.LeaseUntil = &lease + j.UpdatedAt = time.Now().UTC() + a.Public.Status = "processing" + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + claimed = true + return nil + }) + if err != nil || !claimed { + if err == nil { + outcome = "duplicate" + telemetry.Count(ctx, "messenger.jobs", 1, attribute.String("job_type", m.Type), attribute.String("outcome", "duplicate")) + } + return err + } + s.Fault(ctx, "after_claim", m.JobID) + task, cancel := context.WithTimeout(ctx, s.Jobs.Timeout) + defer cancel() + result, e := s.effects(task, j, a) + if e != nil { + if ctx.Err() != nil { + return ctx.Err() + } + finishErr := s.finish(ctx, j, nil, e) + if finishErr == nil { + outcome = "retry" + if j.State == "failed" { + outcome = "failed" + } + telemetry.Count(ctx, "messenger.jobs", 1, attribute.String("job_type", m.Type), attribute.String("outcome", outcome)) + slog.WarnContext(ctx, "job_"+outcome, "job_id", m.JobID) + } + return finishErr + } + s.Fault(ctx, "after_objects", m.JobID) + if err = s.finish(ctx, j, result, nil); err != nil { + return err + } + if !result.Committed { + outcome = "stale" + return nil + } + outcome = "ready" + s.Fault(ctx, "after_result_commit", m.JobID) + telemetry.Count(ctx, "messenger.jobs", 1, attribute.String("job_type", m.Type), attribute.String("outcome", "ready")) + telemetry.Duration(ctx, "messenger.job.ready.duration", time.Since(j.Message.CreatedAt), attribute.String("job_type", m.Type)) + slog.InfoContext(ctx, "job_ready", "job_id", m.JobID) + + return nil +} + +type processed struct { + Committed bool + Key string + Variants []contracts.Variant + Keys map[string]string +} +type permanentError struct{ code string } + +func (e permanentError) Error() string { return e.code } +func (s *Service) effects(ctx context.Context, j *dto.AttachmentJob, a *dto.AttachmentRecord) (*processed, error) { + dir, e := os.MkdirTemp("", "pipeline-*") + if e != nil { + return nil, e + } + defer os.RemoveAll(dir) + key := a.StagingKey + if key == "" { + key = a.Key + } + obj, e := s.Store.Get(ctx, key) + if e != nil { + if objectstore.Missing(e) { + return nil, permanentError{"source_missing"} + } + return nil, e + } + defer obj.Body.Close() + path := filepath.Join(dir, "source") + f, e := os.Create(path) + if e != nil { + return nil, e + } + hash := sha256.New() + n, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(obj.Body, s.Config.MaxBytes+1)) + f.Close() + if e != nil { + return nil, e + } + if n != int64(a.Public.SizeBytes) || fmt.Sprintf("%x", hash.Sum(nil)) != a.Public.Sha256 { + return nil, permanentError{"source_corrupt"} + } + f, e = os.Open(path) + if e != nil { + return nil, e + } + e = s.Validate(ctx, f, string(a.Public.ContentType)) + f.Close() + if e != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, permanentError{"invalid_image"} + } + s.Fault(ctx, "before_objects", j.Message.JobID) + if e := fault.Error("before_objects", j.Message.JobID); e != nil { + if e.Error() == "permanent" { + return nil, permanentError{"invalid_image"} + } + return nil, e + } + imagectx, imageSpan := telemetry.Start(ctx, "image.transform", trace.SpanKindInternal) + converted, e := imageprocessor.Process(imagectx, path, dir, j.Message.PipelineVersion) + telemetry.End(imageSpan, e) + if e != nil { + return nil, e + } + prefix := fmt.Sprintf("results/%s/v%d/", a.Public.Id, j.Message.PipelineVersion) + result := &processed{Key: prefix + "original/" + a.Public.Sha256, Variants: []contracts.Variant{}, Keys: map[string]string{}} + f, e = os.Open(path) + if e != nil { + return nil, e + } + e = s.Store.Put(ctx, result.Key, string(a.Public.ContentType), f, n) + f.Close() + if e != nil { + return nil, e + } + s.Fault(ctx, "after_original_put", j.Message.JobID) + for _, v := range converted { + b, e := os.ReadFile(v.Path) + if e != nil { + return nil, e + } + sum := fmt.Sprintf("%x", sha256.Sum256(b)) + key := prefix + v.Name + "/" + sum + if e = s.Store.Put(ctx, key, v.MIME, bytes.NewReader(b), int64(len(b))); e != nil { + return nil, e + } + result.Keys[v.Name] = key + result.Variants = append(result.Variants, contracts.Variant{Name: contracts.VariantName(v.Name), ContentType: contracts.VariantContentType(v.MIME), Width: v.Width, Height: v.Height, SizeBytes: len(b), Sha256: sum}) + } + return result, nil +} +func (s *Service) finish(ctx context.Context, owned *dto.AttachmentJob, result *processed, cause error) error { + applied := false + err := s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, owned.Message.JobID) + if e != nil { + return e + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if j.Token != owned.Token || j.State != "processing" || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) || a.PipelineVersion != j.Message.PipelineVersion { + return nil + } + if cause != nil { + err := s.retry(ctx, tx, j, a, cause) + if err == nil { + owned.State = j.State + } + return err + } + + if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, "original", result.Key, a.Public.Sha256, int64(a.Public.SizeBytes)); e != nil { + return e + } + for _, v := range result.Variants { + if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, string(v.Name), result.Keys[string(v.Name)], v.Sha256, int64(v.SizeBytes)); e != nil { + return e + } + } + j.State = "ready" + j.ErrorCode = nil + j.LeaseUntil = nil + j.UpdatedAt = time.Now().UTC() + a.Missing = false + a.Key = result.Key + a.VariantKeys = result.Keys + a.Public.Variants = result.Variants + a.Public.Status = "ready" + a.Public.ErrorCode = nil + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + applied = true + return s.Repo.ReadySLO(ctx, tx, j.Message.JobID, j.UpdatedAt) + }) + if result != nil { + result.Committed = err == nil && applied + } + return err +} +func (s *Service) retry(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob, a *dto.AttachmentRecord, cause error) error { + j.Message.Traceparent, j.Message.Tracestate = telemetry.Inject(ctx) + + code := "processing_failed" + var permanent permanentError + terminal := errors.As(cause, &permanent) || j.Message.Attempt >= j.AttemptLimit + if permanent.code != "" { + code = permanent.code + } + if terminal && permanent.code == "" { + code = "retry_exhausted" + } + j.LeaseUntil = nil + j.Token = uuid.New() + j.UpdatedAt = time.Now().UTC() + j.ErrorCode = &code + if terminal { + if e := s.Repo.FailSLO(ctx, tx, j.Message.JobID, j.UpdatedAt); e != nil { + return e + } + j.State = "failed" + a.Public.Status = "failed" + a.Public.ErrorCode = &code + } else { + j.State = "queued" + j.Message.Attempt++ + j.NextAttemptAt = j.UpdatedAt.Add(outbox.Backoff(j.Message.Attempt - 1)) + a.Public.Status = "queued" + a.Public.ErrorCode = nil + if e := s.enqueue(ctx, tx, j); e != nil { + return e + } + } + a.Public.UpdatedAt = j.UpdatedAt + if e := s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + return s.Repo.UpdateTx(ctx, tx, a) +} +func (s *Service) Recover(ctx context.Context) error { + ids, e := s.Repo.ExpiredJobs(ctx) + if e != nil { + return e + } + for _, id := range ids { + e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + if j.State != "processing" || j.LeaseUntil == nil || j.LeaseUntil.After(time.Now()) { + return nil + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if a.PipelineVersion != j.Message.PipelineVersion { + return nil + } + return s.retry(telemetry.Extract(ctx, j.Message.Traceparent, j.Message.Tracestate), tx, j, a, errors.New("lease expired")) + }) + if e != nil { + return e + } + } + dead, e := s.Repo.FailedOutbox(ctx) + if e != nil { + return e + } + for _, id := range dead { + if e = s.PublishFailed(ctx, id); e != nil { + return e + } + } + return nil +} +func (s *Service) PublishFailed(ctx context.Context, id uuid.UUID) error { + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + if j.State != "queued" { + return nil + } + dead, e := s.Repo.PublicationDeadTx(ctx, tx, id, j.Message.Attempt) + if e != nil { + return e + } + if !dead { + return nil + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if a.PipelineVersion != j.Message.PipelineVersion { + return nil + } + if e := s.Repo.FailSLO(ctx, tx, j.Message.JobID, time.Now().UTC()); e != nil { + return e + } + code := "publish_failed" + j.State = "failed" + j.ErrorCode = &code + j.UpdatedAt = time.Now().UTC() + a.Public.Status = "failed" + a.Public.ErrorCode = &code + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + return s.Repo.UpdateTx(ctx, tx, a) + }) +} +func (s *Service) Replay(ctx context.Context, id uuid.UUID) error { + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if j.State != "failed" || a.PipelineVersion != j.Message.PipelineVersion { + return apperr.New(apperr.CodeConflict, "only current failed job can be replayed") + } + source := a.StagingKey + if source == "" { + return errors.New("source unavailable; restore original first") + } + if _, e = s.Store.Head(ctx, source); e != nil { + return errors.New("source unavailable") + } + j.State = "queued" + j.Message.Attempt++ + j.AttemptLimit = j.Message.Attempt + s.Jobs.MaxAttempts - 1 + j.NextAttemptAt = time.Now().UTC() + j.UpdatedAt = j.NextAttemptAt + j.Token = uuid.New() + j.ErrorCode = nil + j.LeaseUntil = nil + a.Public.Status = "queued" + a.Public.ErrorCode = nil + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + if e = s.Repo.AuditTx(ctx, tx, id, "replay"); e != nil { + return e + } + return s.enqueue(ctx, tx, j) + }) +} +func (s *Service) Reprocess(ctx context.Context, id uuid.UUID, version int) error { + if version < 1 || version > 2 { + return errors.New("supported pipeline versions: 1, 2") + } + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + a, e := s.Repo.GetTx(ctx, tx, id) + if e != nil { + return e + } + if version <= a.PipelineVersion || a.Public.Status != "ready" { + return errors.New("requires ready attachment and newer version") + } + a.PipelineVersion = version + a.StagingKey = a.Key + a.Public.Status = "queued" + a.Public.Variants = []contracts.Variant{} + a.VariantKeys = map[string]string{} + a.Public.UpdatedAt = time.Now().UTC() + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + return s.newJob(ctx, tx, a) + }) +} +func (s *Service) Cleanup(ctx context.Context) error { + records, e := s.Repo.RetiredSources(ctx, time.Now().Add(-s.Jobs.Retention)) + if e != nil { + return e + } + for _, record := range records { + // Serialize deletion with operator Restore and the initial upload handoff. + _, unlock, lockErr := s.Repo.Lock(ctx, record.StagingKey) + if lockErr != nil { + return lockErr + } + e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + a, e := s.Repo.GetTx(ctx, tx, record.Public.Id) + if e != nil { + return e + } + if a.StagingKey == "" || a.StagingKey == a.Key && a.Public.Status == "ready" { + return nil + } + if a.Public.Status != "ready" && (a.Public.Status != "failed" || time.Since(a.Public.UpdatedAt) < s.Jobs.Retention) { + return nil + } + if !strings.HasPrefix(a.StagingKey, "results/") { + if e = s.Store.Delete(ctx, a.StagingKey); e != nil { + return e + } + } + a.StagingKey = "" + return s.Repo.UpdateTx(ctx, tx, a) + }) + unlock() + if e != nil { + return e + } + } + return nil +} +func managedPipelineKey(key string) bool { + return strings.HasPrefix(key, "staging/") || strings.HasPrefix(key, "results/") +} diff --git a/service/internal/service/attachment/reconcile.go b/service/internal/service/attachment/reconcile.go new file mode 100644 index 0000000..0ad54ad --- /dev/null +++ b/service/internal/service/attachment/reconcile.go @@ -0,0 +1,228 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" +) + +type ScanResult struct { + Apply bool `json:"apply"` + Objects int `json:"objects"` + Metadata int `json:"metadata"` + Orphans int `json:"orphans"` + Fresh int `json:"fresh_orphans"` + Deleted int `json:"deleted"` + Missing int `json:"missing"` + Marked int `json:"marked_missing"` + Protected int `json:"referenced_objects"` + S3Requests int `json:"s3_requests"` + DBChecks int `json:"db_checks"` + LockAttempts int `json:"lock_attempts"` + DurationMS int64 `json:"duration_ms"` + GraceSeconds float64 `json:"grace_seconds"` +} + +func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult, err error) { + start := time.Now() + result.Apply = apply + result.GraceSeconds = s.Config.Grace.Seconds() + defer func() { result.DurationMS = time.Since(start).Milliseconds() }() + if err = s.Repo.Ping(ctx); err != nil { + return + } + result.S3Requests++ + if err = s.Store.Ping(ctx); err != nil { + return + } + token := "" + for { + result.S3Requests++ + page, e := s.Store.List(ctx, token) + if e != nil { + return result, e + } + for _, obj := range page.Contents { + result.Objects++ + key := aws.ToString(obj.Key) + if !ManagedKey(key) { + continue + } + result.LockAttempts++ + conn, unlock, e := s.Repo.Lock(ctx, key) + if e != nil { + return result, e + } + e = func() error { + defer unlock() + result.DBChecks++ + found, e := s.Repo.Exists(ctx, conn, key) + if e != nil { + return e + } + if found { + result.Protected++ + return nil + } + // Re-read both metadata and object state under the same lock as upload. + result.S3Requests++ + head, e := s.Store.Head(ctx, key) + if objectstore.Missing(e) { + return nil + } + if e != nil { + return e + } + result.Orphans++ + if head.LastModified == nil || time.Since(*head.LastModified) <= s.Config.Grace { + result.Fresh++ + return nil + } + if !apply { + return nil + } + result.DBChecks++ + if e = s.Repo.CheckConn(ctx, conn); e != nil { + return e + } + result.S3Requests++ + if e = s.Store.Delete(ctx, key); e != nil { + return e + } + result.Deleted++ + return nil + }() + if e != nil { + return result, e + } + } + if !aws.ToBool(page.IsTruncated) { + break + } + token = aws.ToString(page.NextContinuationToken) + } + + after := uuid.Nil + for { + result.DBChecks++ + page, e := s.Repo.Page(ctx, after) + if e != nil { + return result, e + } + if len(page) == 0 { + break + } + for _, a := range page { + result.Metadata++ + after = a.Public.Id + if a.Public.Status != "ready" { + continue + } + result.LockAttempts++ + conn, unlock, e := s.Repo.Lock(ctx, a.Key) + if e != nil { + return result, e + } + e = func() error { + defer unlock() + result.S3Requests++ + _, e := s.Store.Head(ctx, a.Key) + if e == nil { + return nil + } + if !objectstore.Missing(e) { + return e + } + result.Missing++ + if apply { + result.DBChecks++ + changed, err := s.Repo.MarkMissing(ctx, conn, a.Public.Id, true) + if err != nil { + return err + } + if changed { + result.Marked++ + } + } + return nil + }() + if e != nil { + return result, e + } + } + } + return result, nil +} +func (s *Service) Restore(ctx context.Context, id uuid.UUID, path string) error { + a, e := s.Repo.Get(ctx, id) + if e != nil { + return e + } + f, e := os.Open(path) + if e != nil { + return e + } + defer f.Close() + hash := sha256.New() + size, e := io.Copy(hash, io.LimitReader(f, s.Config.MaxBytes+1)) + if e != nil { + return e + } + if size != int64(a.Public.SizeBytes) || hex.EncodeToString(hash.Sum(nil)) != a.Public.Sha256 { + return fmt.Errorf("original checksum or size mismatch") + } + if e = s.Validate(ctx, f, string(a.Public.ContentType)); e != nil { + return e + } + conn, unlock, e := s.Repo.Lock(ctx, a.Key) + if e != nil { + return e + } + defer unlock() + if _, e = f.Seek(0, 0); e != nil { + return e + } + if e = s.Store.Put(ctx, a.Key, string(a.Public.ContentType), f, size); e != nil { + return e + } + obj, e := s.Store.Get(ctx, a.Key) + if e != nil { + return e + } + defer obj.Body.Close() + hash.Reset() + n, e := io.Copy(hash, io.LimitReader(obj.Body, s.Config.MaxBytes+1)) + if e != nil { + return e + } + if n != size || hex.EncodeToString(hash.Sum(nil)) != a.Public.Sha256 { + return fmt.Errorf("stored original verification failed") + } + _, e = s.Repo.MarkMissing(ctx, conn, id, false) + if e != nil { + return e + } + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + current, e := s.Repo.GetTx(ctx, tx, id) + if e != nil { + return e + } + if current.Public.Status == "failed" { + current.StagingKey = current.Key + current.Public.UpdatedAt = time.Now().UTC() + return s.Repo.UpdateTx(ctx, tx, current) + } + return nil + }) +} diff --git a/service/internal/service/attachment/service.go b/service/internal/service/attachment/service.go new file mode 100644 index 0000000..47749e4 --- /dev/null +++ b/service/internal/service/attachment/service.go @@ -0,0 +1,169 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "fmt" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "go.opentelemetry.io/otel/trace" + "image" + _ "image/jpeg" + _ "image/png" + "io" + + "os" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "github.com/jackc/pgx/v5" +) + +const Prefix = "originals/" +const StagingPrefix = "staging/" + +type Membership interface { + GetForMember(context.Context, dto.ChatMember) (*dto.Chat, error) +} +type Service struct { + Repo Repository + Store ObjectStore + Config *config.Attachment + chats Membership + tm TransactionManager + outbox Outbox + Jobs *config.Pipeline + uploads, decoders chan struct{} +} + +func New(repo Repository, store ObjectStore, c *config.Attachment, chats Membership) *Service { + return &Service{Repo: repo, Store: store, Config: c, chats: chats, uploads: make(chan struct{}, c.Uploads), decoders: make(chan struct{}, c.Decoders)} +} +func ValidFilename(s string) bool { + if strings.TrimSpace(s) == "" || !utf8.ValidString(s) || utf8.RuneCountInString(s) < 1 || utf8.RuneCountInString(s) > 255 { + return false + } + for _, r := range s { + if r == '/' || r == '\\' || unicode.IsControl(r) { + return false + } + } + return true +} +func (s *Service) Configure(tm TransactionManager, outbox Outbox, jobs *config.Pipeline) { + s.tm = tm + s.outbox = outbox + s.Jobs = jobs +} +func (s *Service) Upload(ctx context.Context, req dto.UploadAttachment) (out *contracts.Attachment, err error) { + ctx, sp := telemetry.Start(ctx, "attachment.upload", trace.SpanKindInternal) + defer func() { telemetry.End(sp, err) }() + if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: req.ChatID}); e != nil { + return nil, e + } + select { + case s.uploads <- struct{}{}: + defer func() { <-s.uploads }() + default: + return nil, apperr.New(apperr.CodeTooManyRequests, "upload capacity exceeded") + } + f, e := os.CreateTemp("", "attachment-*") + if e != nil { + return nil, e + } + defer os.Remove(f.Name()) + defer f.Close() + hash := sha256.New() + size, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(req.Body, s.Config.MaxBytes+1)) + if e != nil { + return nil, apperr.New(apperr.CodeBadRequest, "incomplete upload") + } + if size > s.Config.MaxBytes { + return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large") + } + if e = s.Validate(ctx, f, req.ContentType); e != nil { + return nil, e + } + now := time.Now().UTC().Truncate(time.Microsecond) + id := uuid.New() + key := StagingPrefix + id.String() + a := &dto.AttachmentRecord{Key: key, StagingKey: key, PipelineVersion: 1, VariantKeys: map[string]string{}, Public: contracts.Attachment{Id: id, ChatId: req.ChatID, OwnerId: req.UserID, Filename: req.Filename, ContentType: contracts.AttachmentContentType(req.ContentType), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "queued", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}} + _, unlock, e := s.Repo.Lock(ctx, key) + if e != nil { + return nil, e + } + defer unlock() + if _, e = f.Seek(0, 0); e != nil { + return nil, e + } + if e = s.Store.Put(ctx, key, req.ContentType, f, size); e != nil { + return nil, Unavailable() + } + s.Fault(ctx, "after_staging", id) + if e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + if e := s.Repo.InsertTx(ctx, tx, a); e != nil { + return e + } + return s.newJob(ctx, tx, a) + }); e != nil { + return nil, e + } + s.Fault(ctx, "after_upload_commit", id) + return &a.Public, nil +} +func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error { + if _, e := f.Seek(0, 0); e != nil { + return e + } + cfg, format, e := image.DecodeConfig(f) + if e != nil || ("image/"+format) != mt { + return apperr.New(apperr.CodeUnsupportedMediaType, "invalid image or MIME mismatch") + } + if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > s.Config.MaxPixels/int64(cfg.Height) { + return apperr.New(apperr.CodePayloadTooLarge, "image pixel limit exceeded") + } + select { + case s.decoders <- struct{}{}: + defer func() { <-s.decoders }() + case <-ctx.Done(): + return ctx.Err() + } + if _, e = f.Seek(0, 0); e != nil { + return e + } + if e = validateImage(ctx, f.Name()); e != nil { + return apperr.New(apperr.CodeUnsupportedMediaType, "corrupt image") + } + return ctx.Err() +} +func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*dto.AttachmentRecord, error) { + a, e := s.Repo.Get(ctx, id) + if e != nil { + return nil, e + } + if _, e = s.chats.GetForMember(ctx, dto.ChatMember{UserID: user, ChatID: a.Public.ChatId}); e != nil { + return nil, e + } + if a.Missing { + return nil, Unavailable() + } + return a, nil +} +func Unavailable() error { return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") } +func ManagedKey(key string) bool { + if managedPipelineKey(key) { + return true + } + if !strings.HasPrefix(key, Prefix) { + return false + } + id, e := uuid.Parse(strings.TrimPrefix(key, Prefix)) + return e == nil && key == Prefix+id.String() +} diff --git a/service/internal/service/auth/api.go b/service/internal/service/auth/api.go new file mode 100644 index 0000000..6c9ec6a --- /dev/null +++ b/service/internal/service/auth/api.go @@ -0,0 +1,15 @@ +package auth + +import ( + "context" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type API interface { + Login(ctx context.Context, req dto.Login) (*dto.LoginResult, error) + VerifyAccessToken(ctx context.Context, req dto.Token) (*dto.UserDataInToken, error) + VerifySession(ctx context.Context, token dto.Token) (*dto.UserDataInToken, error) + RefreshTokens(ctx context.Context, req dto.RefreshToken) (*dto.Tokens, error) + Logout(ctx context.Context, req dto.RefreshToken) error +} diff --git a/service/internal/service/auth/interface.go b/service/internal/service/auth/interface.go new file mode 100644 index 0000000..4b2473d --- /dev/null +++ b/service/internal/service/auth/interface.go @@ -0,0 +1,21 @@ +package auth + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type userRepo interface { + GetCredentialsByUsername(ctx context.Context, username string) (*dto.UserCredentials, error) +} + +type refreshTokensRepo interface { + GetByID(ctx context.Context, id uuid.UUID) (*dto.StoredRefreshToken, error) + GetByIDForUpdateTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.StoredRefreshToken, error) + CreateTx(ctx context.Context, tx pgx.Tx, req dto.StoredRefreshToken) error + RevokeTx(ctx context.Context, tx pgx.Tx, req dto.RevokeRefreshToken) error +} diff --git a/service/internal/service/auth/svc.go b/service/internal/service/auth/svc.go new file mode 100644 index 0000000..a91e5be --- /dev/null +++ b/service/internal/service/auth/svc.go @@ -0,0 +1,285 @@ +package auth + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "go.opentelemetry.io/otel/trace" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +var _ API = (*Auth)(nil) + +type Auth struct { + jwt jwtprovider.Provider + tm *txmanager.TransactionManager + encryptor encrypting.Encryptor + + usersRepo userRepo + refreshTokensRepo refreshTokensRepo +} + +func New( + jwt jwtprovider.Provider, + tm *txmanager.TransactionManager, + encryptor encrypting.Encryptor, + usersRepo userRepo, + refreshTokensRepo refreshTokensRepo, +) *Auth { + return &Auth{ + jwt: jwt, + tm: tm, + encryptor: encryptor, + usersRepo: usersRepo, + refreshTokensRepo: refreshTokensRepo, + } +} + +func (a *Auth) Login(ctx context.Context, req dto.Login) (*dto.LoginResult, error) { + ctx, span := telemetry.Start(ctx, "auth.Login", trace.SpanKindInternal) + defer span.End() + if req.Username == "" || req.Password == "" { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid credentials") + } + + credentials, err := a.usersRepo.GetCredentialsByUsername(ctx, req.Username) + if err != nil { + if apperr.CodeFromError(err) == apperr.CodeNotFound { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid credentials") + } + return nil, err + } + if err = a.encryptor.CompareHashAndSecretUsingArgon2id(ctx, credentials.PasswordHash, req.Password); err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, apperr.New(apperr.CodeUnauthorized, "invalid credentials") + } + + var ( + tokens *dto.Tokens + data *dto.UserDataInToken + ) + err = a.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + tokens, err = a.jwt.CreateRefreshAndAccessTokensForUser(credentials.User.ID, role.User) + if err != nil { + return apperr.Wrap(err, apperr.CodeInternal, "failed to generate tokens") + } + data, err = a.verifyToken(tokens.RefreshToken.Token(), false) + if err != nil { + return apperr.Wrap(err, apperr.CodeInternal, "failed to validate generated refresh token") + } + var hash string + hash, err = a.encryptor.EncryptSecretUsingSum256(ctx, tokens.RefreshToken.Payload) + if err != nil { + return apperr.Wrap(err, apperr.CodeInternal, "failed to hash refresh token") + } + stored := dto.StoredRefreshToken{ + ID: data.TokenID, + UserID: data.UserID, + TokenHash: hash, + Role: data.Role, + ExpiresAt: data.ExpiresAt, + CreatedAt: data.IssuedAt, + } + if err = a.refreshTokensRepo.CreateTx(ctx, tx, stored); err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err + } + + return &dto.LoginResult{ + ExpiresAt: data.ExpiresAt, + TokenType: "Bearer", + User: credentials.User, + Tokens: *tokens, + }, nil +} + +func (a *Auth) VerifyAccessToken(ctx context.Context, req dto.Token) (*dto.UserDataInToken, error) { + ctx, span := telemetry.Start(ctx, "auth.VerifyAccessToken", trace.SpanKindInternal) + defer span.End() + if err := ctx.Err(); err != nil { + return nil, err + } + return a.verifyToken(&req, true) +} + +func (a *Auth) VerifySession(ctx context.Context, token dto.Token) (*dto.UserDataInToken, error) { + ctx, span := telemetry.Start(ctx, "auth.VerifySession", trace.SpanKindInternal) + defer span.End() + if err := ctx.Err(); err != nil { + return nil, err + } + data, err := a.verifyTokenClaims(&token) + if err != nil { + return nil, err + } + if data.IsAccess { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid session token") + } + + stored, err := a.refreshTokensRepo.GetByID(ctx, data.TokenID) + if err != nil { + if apperr.CodeFromError(err) == apperr.CodeNotFound { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid session") + } + return nil, err + } + if err = a.validateStoredSession(ctx, token, data, stored); err != nil { + return nil, err + } + return data, nil +} + +func (a *Auth) Logout(ctx context.Context, req dto.RefreshToken) error { + ctx, span := telemetry.Start(ctx, "auth.Logout", trace.SpanKindInternal) + defer span.End() + data, err := a.verifyToken(req.Token(), false) + if err != nil { + return err + } + + return a.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + stored, err := a.activeRefreshTokenTx(ctx, tx, req, data) + if err != nil { + return err + } + return a.refreshTokensRepo.RevokeTx(ctx, tx, dto.RevokeRefreshToken{ + ID: stored.ID, + RevokedAt: time.Now(), + }) + }) +} + +func (a *Auth) RefreshTokens(ctx context.Context, req dto.RefreshToken) (*dto.Tokens, error) { + ctx, span := telemetry.Start(ctx, "auth.RefreshTokens", trace.SpanKindInternal) + defer span.End() + data, err := a.verifyToken(req.Token(), false) + if err != nil { + return nil, err + } + + var tokens *dto.Tokens + err = a.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + stored, err := a.activeRefreshTokenTx(ctx, tx, req, data) + if err != nil { + return err + } + if err = a.refreshTokensRepo.RevokeTx(ctx, tx, dto.RevokeRefreshToken{ + ID: stored.ID, + RevokedAt: time.Now(), + }); err != nil { + return err + } + tokens, err = a.createTokensTx(ctx, tx, stored.UserID, stored) + return err + }) + if err != nil { + return nil, err + } + return tokens, nil +} + +func (a *Auth) verifyToken(token *dto.Token, isAccess bool) (*dto.UserDataInToken, error) { + data, err := a.verifyTokenClaims(token) + if err != nil { + return nil, err + } + if data.IsAccess != isAccess || data.Role != role.User { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid token") + } + return data, nil +} + +func (a *Auth) verifyTokenClaims(token *dto.Token) (*dto.UserDataInToken, error) { + data, err := a.jwt.ParseVerified(token) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeUnauthorized, "invalid token") + } + if err = data.Validate(); err != nil { + return nil, apperr.Wrap(err, apperr.CodeUnauthorized, "invalid token claims") + } + now := time.Now() + if data.IssuedAt.After(now) || !data.ExpiresAt.After(now) { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid token") + } + return data, nil +} + +func (a *Auth) activeRefreshTokenTx(ctx context.Context, tx pgx.Tx, token dto.RefreshToken, data *dto.UserDataInToken) (*dto.StoredRefreshToken, error) { + ctx, span := telemetry.Start(ctx, "auth.activeRefreshTokenTx", trace.SpanKindInternal) + defer span.End() + stored, err := a.refreshTokensRepo.GetByIDForUpdateTx(ctx, tx, data.TokenID) + if err != nil { + if apperr.CodeFromError(err) == apperr.CodeNotFound { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid refresh token") + } + return nil, err + } + + // Recheck expiry after acquiring the row lock: a concurrent rotation may have held it. + if err = a.validateStoredSession(ctx, dto.Token(token), data, stored); err != nil { + return nil, err + } + return stored, nil +} + +func (a *Auth) validateStoredSession(ctx context.Context, token dto.Token, data *dto.UserDataInToken, stored *dto.StoredRefreshToken) error { + ctx, span := telemetry.Start(ctx, "auth.validateStoredSession", trace.SpanKindInternal) + defer span.End() + now := time.Now() + if stored.RevokedAt != nil || !stored.ExpiresAt.After(now) || !data.ExpiresAt.After(now) || + stored.UserID != data.UserID || stored.Role != data.Role { + return apperr.New(apperr.CodeUnauthorized, "invalid session") + } + if err := a.encryptor.CompareHashAndSecretUsingSum256(ctx, stored.TokenHash, token.Payload); err != nil { + return apperr.New(apperr.CodeUnauthorized, "invalid session") + } + return nil +} + +func (a *Auth) createTokensTx(ctx context.Context, tx pgx.Tx, userID uuid.UUID, previous *dto.StoredRefreshToken) (*dto.Tokens, error) { + ctx, span := telemetry.Start(ctx, "auth.createTokensTx", trace.SpanKindInternal) + defer span.End() + tokens, err := a.jwt.CreateRefreshAndAccessTokensForUser(userID, role.User) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to generate tokens") + } + data, err := a.verifyToken(tokens.RefreshToken.Token(), false) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to validate generated refresh token") + } + hash, err := a.encryptor.EncryptSecretUsingSum256(ctx, tokens.RefreshToken.Payload) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to hash refresh token") + } + stored := dto.StoredRefreshToken{ + ID: data.TokenID, + UserID: data.UserID, + TokenHash: hash, + Role: data.Role, + ExpiresAt: data.ExpiresAt, + CreatedAt: data.IssuedAt, + } + if previous != nil { + stored.UserAgent = previous.UserAgent + stored.IPAddress = previous.IPAddress + } + if err = a.refreshTokensRepo.CreateTx(ctx, tx, stored); err != nil { + return nil, err + } + return tokens, nil +} diff --git a/service/internal/service/chat/api.go b/service/internal/service/chat/api.go new file mode 100644 index 0000000..cae5a2d --- /dev/null +++ b/service/internal/service/chat/api.go @@ -0,0 +1,15 @@ +package chat + +import ( + "context" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type API interface { + CreateChat(ctx context.Context, req dto.CreateChat) (*dto.Chat, error) + GetForMember(ctx context.Context, req dto.ChatMember) (*dto.Chat, error) + ListForMember(ctx context.Context, req dto.ChatFilters) (*dto.Page[dto.Chat], error) + CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.Message, error) + ListForChat(ctx context.Context, req dto.MessageFilters) (*dto.Page[dto.Message], error) +} diff --git a/service/internal/service/chat/idempotency.go b/service/internal/service/chat/idempotency.go new file mode 100644 index 0000000..13fa962 --- /dev/null +++ b/service/internal/service/chat/idempotency.go @@ -0,0 +1,54 @@ +package chat + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "time" +) + +// CreateIdempotentTx performs the message use case inside an existing transaction. +func CreateIdempotentTx(ctx context.Context, tx pgx.Tx, repo messagesRepo, req dto.CreateMessage) (*dto.Message, error) { + scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey + if err := repo.LockKeyTx(ctx, tx, scope); err != nil { + return nil, err + } + canonical := struct { + Text string `json:"text"` + AttachmentIDs []uuid.UUID `json:"attachment_ids"` + }{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)} + encoded, err := json.Marshal(canonical) + if err != nil { + return nil, err + } + hash := sha256.Sum256(encoded) + replay, err := repo.ReplayTx(ctx, tx, req) + if err != nil { + return nil, err + } + if replay != nil { + if replay.HashVersion == 1 && len(req.AttachmentIDs) == 0 { + hash = sha256.Sum256([]byte(req.Text)) + } else if replay.HashVersion != 2 { + return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") + } + if !bytes.Equal(hash[:], replay.Hash) { + return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") + } + return repo.MessageTx(ctx, tx, replay.MessageID) + } + now := time.Now().UTC().Truncate(time.Microsecond) + message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: now} + if err = repo.InsertMessageTx(ctx, tx, message); err != nil { + return nil, err + } + if err = repo.SaveKeyTx(ctx, tx, req, hash[:], message.ID, now.Add(24*time.Hour)); err != nil { + return nil, err + } + return &message, nil +} diff --git a/service/internal/service/chat/interface.go b/service/internal/service/chat/interface.go new file mode 100644 index 0000000..17c7742 --- /dev/null +++ b/service/internal/service/chat/interface.go @@ -0,0 +1,33 @@ +package chat + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type chatsRepo interface { + CreateTx(ctx context.Context, tx pgx.Tx, req dto.Chat) error + GetForMember(ctx context.Context, req dto.ChatMember) (*dto.Chat, error) + ListForMember(ctx context.Context, userID uuid.UUID, page dto.PageQuery) ([]dto.Chat, error) +} + +type messagesRepo interface { + AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) + LockKeyTx(context.Context, pgx.Tx, string) error + ReplayTx(context.Context, pgx.Tx, dto.CreateMessage) (*dto.MessageReplay, error) + MessageTx(context.Context, pgx.Tx, uuid.UUID) (*dto.Message, error) + InsertMessageTx(context.Context, pgx.Tx, dto.Message) error + SaveKeyTx(context.Context, pgx.Tx, dto.CreateMessage, []byte, uuid.UUID, time.Time) error + GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) + ListForChat(ctx context.Context, chatID uuid.UUID, page dto.PageQuery) ([]dto.Message, error) +} + +type chatsCache interface { + Create(ctx context.Context, req dto.Chat) error + Get(ctx context.Context, chatID uuid.UUID) (*dto.Chat, error) +} diff --git a/service/internal/service/chat/svc.go b/service/internal/service/chat/svc.go new file mode 100644 index 0000000..ebe847a --- /dev/null +++ b/service/internal/service/chat/svc.go @@ -0,0 +1,303 @@ +package chat + +import ( + "bytes" + "context" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "go.opentelemetry.io/otel/trace" + "log/slog" + "slices" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/redis/go-redis/v9" + "golang.org/x/sync/singleflight" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" +) + +var _ API = (*Chat)(nil) + +const chatLoadTimeout = 2 * time.Second + +type Chat struct { + tm *txmanager.TransactionManager + chatsRepo chatsRepo + messagesRepo messagesRepo + chatsCache chatsCache + loads singleflight.Group +} + +func New(tm *txmanager.TransactionManager, chatsRepo chatsRepo, messagesRepo messagesRepo, chatsCache chatsCache) *Chat { + return &Chat{ + tm: tm, + chatsRepo: chatsRepo, + messagesRepo: messagesRepo, + chatsCache: chatsCache, + } +} + +func (c *Chat) CreateChat(ctx context.Context, req dto.CreateChat) (*dto.Chat, error) { + ctx, span := telemetry.Start(ctx, "chat.CreateChat", trace.SpanKindInternal) + defer span.End() + seen := make(map[uuid.UUID]struct{}, len(req.MemberIDs)) + for _, id := range req.MemberIDs { + if id == uuid.Nil { + return nil, apperr.New(apperr.CodeBadRequest, "invalid chat member") + } + if _, exists := seen[id]; exists { + return nil, apperr.New(apperr.CodeBadRequest, "duplicate chat member") + } + seen[id] = struct{}{} + } + + if _, included := seen[req.CreatorID]; !included { + return nil, apperr.New(apperr.CodeBadRequest, "creator must be a chat member") + } + + members := slices.Clone(req.MemberIDs) + slices.SortFunc(members, func(a, b uuid.UUID) int { + return bytes.Compare(a[:], b[:]) + }) + chat := dto.Chat{ + ID: uuid.New(), + Title: req.Title, + MemberIDs: members, + CreatedAt: time.Now().Truncate(time.Microsecond), + } + + if err := c.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + return c.chatsRepo.CreateTx(ctx, tx, chat) + }); err != nil { + return nil, err + } + + return &chat, nil +} + +func (c *Chat) GetForMember(ctx context.Context, req dto.ChatMember) (*dto.Chat, error) { + ctx, span := telemetry.Start(ctx, "chat.GetForMember", trace.SpanKindInternal) + defer span.End() + if err := ctx.Err(); err != nil { + return nil, err + } + if chat, err := c.cachedChat(ctx, req.ChatID); err == nil { + return chatForMember(chat, req.UserID) + } + + // Repository results depend on membership: never share a user's 404 with + // another user requesting the same chat. + key := req.ChatID.String() + ":" + req.UserID.String() + result := c.loads.DoChan(key, func() (any, error) { + // One client disconnecting must not cancel the other clients' load. + loadCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), chatLoadTimeout) + defer cancel() + + // A previous load may have populated Redis after our initial miss. + if chat, err := c.cachedChat(loadCtx, req.ChatID); err == nil { + return chatForMember(chat, req.UserID) + } + + slog.InfoContext(loadCtx, "chat_db_lookup", "chat_id", req.ChatID) + chat, err := c.chatsRepo.GetForMember(loadCtx, req) + if err != nil { + return nil, err + } + if err = c.chatsCache.Create(loadCtx, *chat); err != nil { + slog.WarnContext(loadCtx, "chat_cache_write_failed", "chat_id", req.ChatID, "error", err) + } + return chat, nil + }) + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case loaded := <-result: + if err := ctx.Err(); err != nil { + return nil, err + } + if loaded.Err != nil { + return nil, loaded.Err + } + return chatForMember(loaded.Val.(*dto.Chat), req.UserID) + } +} + +func (c *Chat) cachedChat(ctx context.Context, chatID uuid.UUID) (*dto.Chat, error) { + ctx, span := telemetry.Start(ctx, "chat.cachedChat", trace.SpanKindInternal) + defer span.End() + chat, err := c.chatsCache.Get(ctx, chatID) + switch { + case err == nil: + slog.DebugContext(ctx, "chat_cache_hit", "chat_id", chatID) + case errors.Is(err, redis.Nil): + slog.DebugContext(ctx, "chat_cache_miss", "chat_id", chatID) + default: + slog.WarnContext(ctx, "chat_cache_read_error", "chat_id", chatID, "error", err) + } + return chat, err +} + +func chatForMember(chat *dto.Chat, userID uuid.UUID) (*dto.Chat, error) { + if !slices.Contains(chat.MemberIDs, userID) { + return nil, apperr.New(apperr.CodeNotFound, "chat not found") + } + return chat, nil +} + +func (c *Chat) ListForMember(ctx context.Context, req dto.ChatFilters) (*dto.Page[dto.Chat], error) { + ctx, span := telemetry.Start(ctx, "chat.ListForMember", trace.SpanKindInternal) + defer span.End() + limit, err := pageLimit(req.Limit) + if err != nil { + return nil, err + } + + query := dto.PageQuery{Limit: limit + 1} + if req.AfterID != nil { + cursor, err := c.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: *req.AfterID}) + if err != nil { + return nil, cursorError(err) + } + query.After = &dto.PagePosition{CreatedAt: cursor.CreatedAt, ID: cursor.ID} + } + + items, err := c.chatsRepo.ListForMember(ctx, req.UserID, query) + if err != nil { + return nil, err + } + + return makePage(items, limit, func(chat dto.Chat) uuid.UUID { return chat.ID }), nil +} + +func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.Message, error) { + ctx, span := telemetry.Start(ctx, "chat.CreateMessage", trace.SpanKindInternal) + defer span.End() + if _, err := c.GetForMember(ctx, dto.ChatMember{UserID: req.SenderID, ChatID: req.ChatID}); err != nil { + return nil, err + } + + if err := validateMessage(req); err != nil { + return nil, err + } + req.AttachmentIDs = append([]uuid.UUID{}, req.AttachmentIDs...) + + var message *dto.Message + err := c.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + if len(req.AttachmentIDs) > 0 { + attachments, err := c.messagesRepo.AttachmentsForMessageTx(ctx, tx, req.AttachmentIDs) + if err != nil { + return err + } + if len(attachments) != len(req.AttachmentIDs) { + return apperr.New(apperr.CodeNotFound, "attachment not found") + } + for _, attachment := range attachments { + if attachment.ChatID != req.ChatID { + return apperr.New(apperr.CodeNotFound, "attachment not found") + } + if attachment.Status == "failed" { + return apperr.New(apperr.CodeConflict, "failed attachment") + } + if attachment.Missing { + return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") + } + } + } + + var err error + message, err = CreateIdempotentTx(ctx, tx, c.messagesRepo, req) + return err + }) + return message, err +} + +func (c *Chat) ListForChat(ctx context.Context, req dto.MessageFilters) (*dto.Page[dto.Message], error) { + ctx, span := telemetry.Start(ctx, "chat.ListForChat", trace.SpanKindInternal) + defer span.End() + if _, err := c.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: req.ChatID}); err != nil { + return nil, err + } + + limit, err := pageLimit(req.Limit) + if err != nil { + return nil, err + } + + query := dto.PageQuery{Limit: limit + 1} + if req.AfterID != nil { + cursor, err := c.messagesRepo.GetByIDForChat(ctx, req.ChatID, *req.AfterID) + if err != nil { + return nil, cursorError(err) + } + query.After = &dto.PagePosition{CreatedAt: cursor.CreatedAt, ID: cursor.ID} + } + + items, err := c.messagesRepo.ListForChat(ctx, req.ChatID, query) + if err != nil { + return nil, err + } + + return makePage(items, limit, func(message dto.Message) uuid.UUID { return message.ID }), nil +} + +func pageLimit(limit *int) (int, error) { + size := 50 + if limit != nil { + size = *limit + } + if size < 1 || size > 100 { + return 0, apperr.New(apperr.CodeBadRequest, "invalid limit") + } + return size, nil +} + +func cursorError(err error) error { + if apperr.CodeFromError(err) == apperr.CodeNotFound { + return apperr.New(apperr.CodeBadRequest, "invalid cursor") + } + return err +} + +func makePage[T any](items []T, limit int, id func(T) uuid.UUID) *dto.Page[T] { + if items == nil { + items = make([]T, 0) + } + page := dto.Page[T]{Items: items} + if len(items) > limit { + page.Items = items[:limit] + cursor := id(page.Items[limit-1]) + page.NextCursorID = &cursor + } + return &page +} + +// Content rules belong to the service so non-HTTP callers cannot bypass them. +func validateMessage(req dto.CreateMessage) error { + if !utf8.ValidString(req.Text) || strings.ContainsRune(req.Text, '\x00') || utf8.RuneCountInString(req.Text) > 4000 { + return apperr.New(apperr.CodeBadRequest, "invalid message text") + } + if len(req.AttachmentIDs) > 10 { + return apperr.New(apperr.CodeBadRequest, "too many attachments") + } + seen := make(map[uuid.UUID]struct{}, len(req.AttachmentIDs)) + for _, id := range req.AttachmentIDs { + if id == uuid.Nil { + return apperr.New(apperr.CodeBadRequest, "invalid attachment id") + } + if _, ok := seen[id]; ok { + return apperr.New(apperr.CodeBadRequest, "duplicate attachment id") + } + seen[id] = struct{}{} + } + if strings.TrimSpace(req.Text) == "" && len(req.AttachmentIDs) == 0 { + return apperr.New(apperr.CodeBadRequest, "message requires text or an attachment") + } + return nil +} diff --git a/service/internal/service/observation/service.go b/service/internal/service/observation/service.go new file mode 100644 index 0000000..98790b8 --- /dev/null +++ b/service/internal/service/observation/service.go @@ -0,0 +1,64 @@ +// Package observation evaluates the lab SLO; storage only persists facts and decisions. +package observation + +import ( + "context" + "github.com/google/uuid" + "log/slog" + "time" +) + +type Sample struct { + ID uuid.UUID + Accepted, Deadline time.Time + Ready *time.Time + Failed *time.Time +} +type Snapshot struct { + Queued, Outbox int64 + Locked int64 + Oldest float64 + Good, Bad int64 +} +type Repository interface { + Pending(context.Context) ([]Sample, error) + Decide(context.Context, uuid.UUID, string, time.Time) error + Snapshot(context.Context) (Snapshot, error) +} +type Service struct { + Repo Repository + Publish func(Snapshot) +} + +func (s *Service) Observe(ctx context.Context) error { + rows, e := s.Repo.Pending(ctx) + if e != nil { + return e + } + now := time.Now().UTC() + for _, r := range rows { + outcome := "" + if r.Failed != nil { + outcome = "bad" + } else if r.Ready != nil { + outcome = "bad" + if !r.Ready.After(r.Deadline) { + outcome = "good" + } + } else if !now.Before(r.Deadline) { + outcome = "bad" + } + if outcome != "" { + if e = s.Repo.Decide(ctx, r.ID, outcome, now); e != nil { + return e + } + } + } + state, e := s.Repo.Snapshot(ctx) + if e != nil { + return e + } + s.Publish(state) + slog.DebugContext(ctx, "observation_updated") + return nil +} diff --git a/service/internal/service/outbox/api.go b/service/internal/service/outbox/api.go new file mode 100644 index 0000000..aaac1a7 --- /dev/null +++ b/service/internal/service/outbox/api.go @@ -0,0 +1,12 @@ +package outbox + +import ( + "context" + + "github.com/jackc/pgx/v5" +) + +type API interface { + CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error) + Create(context.Context, CreateMessageReq) (CreateMessageRes, error) +} diff --git a/service/internal/service/outbox/models.go b/service/internal/service/outbox/models.go new file mode 100644 index 0000000..fc90e0e --- /dev/null +++ b/service/internal/service/outbox/models.go @@ -0,0 +1,115 @@ +package outbox + +import ( + "time" + + "github.com/google/uuid" +) + +type CreateMessageReq struct { + ID uuid.UUID + CreatedAt time.Time + Transport Transport + AggregateType string + AggregateID uuid.UUID + MessageType string + IdempotencyKey *string + SchemaVersion int + CausationID *uuid.UUID + Metadata map[string]string + Payload []byte + MaxAttempts int + Status Status +} + +type CreateMessageRes struct { + ID uuid.UUID + CreatedAt time.Time +} + +type MarkPublishedMessageReq struct { + ID uuid.UUID + UpdatedAt time.Time + PublishedAt time.Time +} + +type MarkFailedMessageReq struct { + ID uuid.UUID + UpdatedAt time.Time + NextAttemptAt time.Time + LastError string +} + +type MarkDeadLetterMessageReq struct { + ID uuid.UUID + UpdatedAt time.Time + DeadLetterAt time.Time + LastError string +} + +type AcquireMessagesForProcessingReq struct { + Limit uint + LockDuration time.Duration +} + +type AcquiredMessage struct { + Token uuid.UUID + ID uuid.UUID + Transport Transport + AggregateType string + AggregateID uuid.UUID + MessageType string + SchemaVersion int + CorrelationID uuid.UUID + CausationID *uuid.UUID + Status Status + Metadata map[string]string + Payload []byte + Attempts int + MaxAttempts int + LastError *string + LockedAt *time.Time + LockedUntil *time.Time + NextAttemptAt *time.Time + PublishedAt *time.Time + DeadLetterAt *time.Time + CreatedAt time.Time + UpdatedAt *time.Time +} + +type RecoverStuckMessagesAsFailedReq struct { + UpdatedAt time.Time + NextAttemptAt time.Time + LastError string + Limit uint +} + +type RecoverStuckMessagesAsDeadLetterReq struct { + UpdatedAt time.Time + DeadLetterAt time.Time + LastError string + Limit uint +} + +type DeliveryMessage struct { + ID uuid.UUID + Transport Transport + AggregateType string + AggregateID uuid.UUID + MessageType string + SchemaVersion int + CorrelationID uuid.UUID + CausationID *uuid.UUID + Payload []byte + Metadata map[string]string +} + +type ProcessOnceResult struct { + RecoveredAsFailed int64 + RecoveredAsDeadLetter int64 + + Acquired int + Published int + Failed int + DeadLettered int +} diff --git a/service/internal/service/outbox/status.go b/service/internal/service/outbox/status.go new file mode 100644 index 0000000..c7a1d3a --- /dev/null +++ b/service/internal/service/outbox/status.go @@ -0,0 +1,16 @@ +package outbox + +const ( + StatusCreated Status = "created" + StatusPending Status = "pending" + StatusProcessing Status = "processing" + StatusPublished Status = "published" + StatusFailed Status = "failed" + StatusDeadLetter Status = "dead_letter" +) + +type Status string + +func (s Status) String() string { + return string(s) +} diff --git a/service/internal/service/outbox/svc.go b/service/internal/service/outbox/svc.go new file mode 100644 index 0000000..7dd7faa --- /dev/null +++ b/service/internal/service/outbox/svc.go @@ -0,0 +1,114 @@ +// Adapted from mamont-optics/application/internal/service/outbox. +// Retains transactional creation, batch acquisition, bounded retries and lease recovery. +package outbox + +import ( + "context" + "encoding/json" + "errors" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "math/rand/v2" + "time" +) + +type Repository interface { + CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error) + AcquireManyForProcessing(context.Context, AcquireMessagesForProcessingReq) ([]AcquiredMessage, error) + Expired(context.Context) ([]AcquiredMessage, error) + Update(context.Context, AcquiredMessage, Status, time.Time, string) error +} +type TransportPublisher interface { + Publish(context.Context, DeliveryMessage) error +} +type TransactionManager interface { + RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error +} +type Svc struct { + repo Repository + tm TransactionManager + publisher TransportPublisher + OnDead func(context.Context, uuid.UUID) error +} + +func New(repo Repository, tm TransactionManager, publisher TransportPublisher) *Svc { + return &Svc{repo: repo, tm: tm, publisher: publisher} +} +func (s *Svc) CreateTx(ctx context.Context, tx pgx.Tx, r CreateMessageReq) (CreateMessageRes, error) { + if r.ID == uuid.Nil { + r.ID = uuid.New() + } + if r.CreatedAt.IsZero() { + r.CreatedAt = time.Now().UTC() + } + if r.Status == "" { + r.Status = StatusPending + } + if r.MaxAttempts == 0 { + r.MaxAttempts = 5 + } + return s.repo.CreateTx(ctx, tx, r) +} +func (s *Svc) Create(ctx context.Context, r CreateMessageReq) (res CreateMessageRes, err error) { + err = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { var e error; res, e = s.CreateTx(ctx, tx, r); return e }) + return +} +func Backoff(attempt int) time.Duration { + n := min(max(attempt-1, 0), 5) + cap := min(time.Second*time.Duration(1<= m.MaxAttempts { + status = StatusDeadLetter + } + if err := s.repo.Update(ctx, m, status, time.Now().UTC().Add(Backoff(m.Attempts)), code); err != nil { + return err + } + if status == StatusDeadLetter && s.OnDead != nil { + var j struct { + JobID uuid.UUID `json:"job_id"` + } + if err := json.Unmarshal(m.Payload, &j); err != nil { + return err + } + return s.OnDead(ctx, j.JobID) + } + return nil +} diff --git a/service/internal/service/outbox/transport.go b/service/internal/service/outbox/transport.go new file mode 100644 index 0000000..a92608c --- /dev/null +++ b/service/internal/service/outbox/transport.go @@ -0,0 +1,7 @@ +package outbox + +type Transport string + +const TransportAMQP Transport = "amqp" + +func (t Transport) String() string { return string(t) } diff --git a/service/internal/service/user/api.go b/service/internal/service/user/api.go new file mode 100644 index 0000000..65329cf --- /dev/null +++ b/service/internal/service/user/api.go @@ -0,0 +1,15 @@ +package user + +import ( + "context" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type API interface { + Create(ctx context.Context, req dto.CreateUser) (*dto.User, error) + GetByID(ctx context.Context, id uuid.UUID) (*dto.User, error) + GetByUsername(ctx context.Context, username string) (*dto.User, error) +} diff --git a/service/internal/service/user/interface.go b/service/internal/service/user/interface.go new file mode 100644 index 0000000..4eb78b5 --- /dev/null +++ b/service/internal/service/user/interface.go @@ -0,0 +1,15 @@ +package user + +import ( + "context" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type userRepo interface { + Create(ctx context.Context, req dto.CreateStoredUser) error + GetByID(ctx context.Context, id uuid.UUID) (*dto.User, error) + GetByUsername(ctx context.Context, username string) (*dto.User, error) +} diff --git a/service/internal/service/user/svc.go b/service/internal/service/user/svc.go new file mode 100644 index 0000000..966329f --- /dev/null +++ b/service/internal/service/user/svc.go @@ -0,0 +1,75 @@ +package user + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "go.opentelemetry.io/otel/trace" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting" +) + +var ( + _ API = (*User)(nil) +) + +type User struct { + encryptor encrypting.Encryptor + usersRepo userRepo +} + +func New(encryptor encrypting.Encryptor, usersRepo userRepo) *User { + return &User{encryptor: encryptor, usersRepo: usersRepo} +} + +func (u *User) Create(ctx context.Context, req dto.CreateUser) (*dto.User, error) { + ctx, span := telemetry.Start(ctx, "user.Create", trace.SpanKindInternal) + defer span.End() + hash, err := u.encryptor.EncryptSecretUsingArgon2id(ctx, req.Password) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to hash password") + } + + user := dto.User{ + ID: uuid.New(), + Username: req.Username, + DisplayName: req.DisplayName, + CreatedAt: time.Now().Truncate(time.Microsecond), + } + if err = u.usersRepo.Create(ctx, dto.CreateStoredUser{ + ID: user.ID, + Username: user.Username, + DisplayName: user.DisplayName, + PasswordHash: hash, + CreatedAt: user.CreatedAt, + UpdatedAt: user.CreatedAt, + }); err != nil { + return nil, err + } + + return &user, nil +} + +func (u *User) GetByID(ctx context.Context, id uuid.UUID) (*dto.User, error) { + ctx, span := telemetry.Start(ctx, "user.GetByID", trace.SpanKindInternal) + defer span.End() + user, err := u.usersRepo.GetByID(ctx, id) + if err != nil { + return nil, err + } + return user, nil +} + +func (u *User) GetByUsername(ctx context.Context, username string) (*dto.User, error) { + ctx, span := telemetry.Start(ctx, "user.GetByUsername", trace.SpanKindInternal) + defer span.End() + user, err := u.usersRepo.GetByUsername(ctx, username) + if err != nil { + return nil, err + } + return user, nil +} diff --git a/service/migrations/001_create_users_table.sql b/service/migrations/001_create_users_table.sql new file mode 100644 index 0000000..60f730a --- /dev/null +++ b/service/migrations/001_create_users_table.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS users +( + id UUID PRIMARY KEY, + name VARCHAR(100) NOT NULL, + username VARCHAR(32) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_name_not_blank CHECK (name !~ '^[[:space:]]*$'), + CONSTRAINT users_password_hash_not_empty CHECK (password_hash <> '') +); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS users; diff --git a/service/migrations/002_create_refresh_tokens_table.sql b/service/migrations/002_create_refresh_tokens_table.sql new file mode 100644 index 0000000..cbc7308 --- /dev/null +++ b/service/migrations/002_create_refresh_tokens_table.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS refresh_tokens +( + id UUID PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users (id), + token_hash TEXT NOT NULL, + user_agent TEXT, + ip_address INET, + role VARCHAR(16) NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_refresh_user_id ON refresh_tokens (user_id); +CREATE INDEX IF NOT EXISTS idx_refresh_expires ON refresh_tokens (expires_at); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS refresh_tokens; \ No newline at end of file diff --git a/service/migrations/003_create_chats_table.sql b/service/migrations/003_create_chats_table.sql new file mode 100644 index 0000000..b45df91 --- /dev/null +++ b/service/migrations/003_create_chats_table.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS chats +( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT chats_title_not_blank CHECK (title !~ '^[[:space:]]*$') +); + +CREATE INDEX IF NOT EXISTS idx_chats_created_at_id ON chats (created_at, id); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS chats; diff --git a/service/migrations/004_create_chat_members_table.sql b/service/migrations/004_create_chat_members_table.sql new file mode 100644 index 0000000..6413198 --- /dev/null +++ b/service/migrations/004_create_chat_members_table.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS chat_members +( + chat_id UUID NOT NULL, + user_id UUID NOT NULL, + + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE NO ACTION, + CONSTRAINT chat_members_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE NO ACTION +); + +CREATE INDEX IF NOT EXISTS idx_chat_members_user_id_chat_id ON chat_members (user_id, chat_id); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS chat_members; diff --git a/service/migrations/005_create_messages_table.sql b/service/migrations/005_create_messages_table.sql new file mode 100644 index 0000000..1ab0d7d --- /dev/null +++ b/service/migrations/005_create_messages_table.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS messages +( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + sender_id UUID NOT NULL, + text VARCHAR(4000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT messages_text_not_blank CHECK (text !~ '^[[:space:]]*$'), + CONSTRAINT messages_sender_member_fk FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE NO ACTION +); + +CREATE INDEX IF NOT EXISTS idx_messages_chat_id_created_at_id ON messages (chat_id, created_at, id); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS messages; diff --git a/service/migrations/006_message_idempotency.sql b/service/migrations/006_message_idempotency.sql new file mode 100644 index 0000000..8e48e9b --- /dev/null +++ b/service/migrations/006_message_idempotency.sql @@ -0,0 +1,14 @@ +CREATE TABLE message_idempotency ( + user_id UUID NOT NULL REFERENCES users(id), + method TEXT NOT NULL CHECK (method = 'POST'), + chat_id UUID NOT NULL REFERENCES chats(id), + key TEXT NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash BYTEA NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id), + expires_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (user_id, method, chat_id, key) +); +CREATE INDEX message_idempotency_expiry ON message_idempotency(expires_at); + +---- create above / drop below ---- +DROP TABLE message_idempotency; diff --git a/service/migrations/007_attachments.sql b/service/migrations/007_attachments.sql new file mode 100644 index 0000000..050c686 --- /dev/null +++ b/service/migrations/007_attachments.sql @@ -0,0 +1,44 @@ +CREATE TABLE attachments +( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id), + owner_id UUID NOT NULL, + filename TEXT NOT NULL, + object_key TEXT NOT NULL UNIQUE, + content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + integrity_missing BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + UNIQUE (id, chat_id), + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id) +); +CREATE INDEX attachments_scan ON attachments (id); +ALTER TABLE messages + DROP CONSTRAINT messages_text_not_blank; +ALTER TABLE messages + ADD CONSTRAINT messages_id_chat UNIQUE (id, chat_id); +CREATE TABLE message_attachments +( + message_id UUID NOT NULL, + chat_id UUID NOT NULL, + attachment_id UUID NOT NULL, + position SMALLINT NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position), + FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id), + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id) +); +ALTER TABLE message_idempotency + ADD COLUMN hash_version SMALLINT NOT NULL DEFAULT 1; + +---- create above / drop below ---- +ALTER TABLE message_idempotency + DROP COLUMN hash_version; +DROP TABLE message_attachments; +ALTER TABLE messages + DROP CONSTRAINT messages_id_chat; +ALTER TABLE messages + ADD CONSTRAINT messages_text_not_blank CHECK (text !~ '^[[:space:]]*$'); +DROP TABLE attachments; diff --git a/service/migrations/008_async_pipeline.sql b/service/migrations/008_async_pipeline.sql new file mode 100644 index 0000000..4607e20 --- /dev/null +++ b/service/migrations/008_async_pipeline.sql @@ -0,0 +1,41 @@ +ALTER TABLE attachments + ADD COLUMN status text NOT NULL DEFAULT 'queued' CHECK(status IN ('queued','processing','ready','failed')), + ADD COLUMN error_code text, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1, + ADD COLUMN staging_key text NOT NULL DEFAULT '', + ADD COLUMN variants jsonb NOT NULL DEFAULT '[]', + ADD COLUMN variant_keys jsonb NOT NULL DEFAULT '{}'; +UPDATE attachments SET staging_key=object_key; +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL, attempt integer NOT NULL CHECK(attempt>0), + attempt_limit integer NOT NULL, state text NOT NULL, + token uuid NOT NULL, lease_until timestamptz, next_attempt_at timestamptz NOT NULL, + error_code text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL, + UNIQUE(attachment_id,pipeline_version) +); +CREATE INDEX jobs_recovery ON attachment_jobs(state,lease_until); +CREATE TABLE outbox ( + id uuid PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id), + payload jsonb NOT NULL, status text NOT NULL, + attempts integer NOT NULL DEFAULT 0, max_attempts integer NOT NULL, + token uuid NOT NULL, locked_until timestamptz, next_attempt_at timestamptz NOT NULL, + last_error text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL +); +CREATE INDEX outbox_delivery ON outbox(status,next_attempt_at); +CREATE TABLE job_audit ( + id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id), + action text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +-- Existing lab4 originals become durable sources for the first asynchronous run. +INSERT INTO attachment_jobs + SELECT gen_random_uuid(),id,1,1,5,'queued',gen_random_uuid(),NULL,clock_timestamp(),NULL,created_at,clock_timestamp() FROM attachments; +INSERT INTO outbox + SELECT gen_random_uuid(),id,jsonb_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',1,'attempt',1,'created_at',created_at), + 'pending',0,5,gen_random_uuid(),NULL,clock_timestamp(),NULL,clock_timestamp(),clock_timestamp() FROM attachment_jobs; + +---- create above / drop below ---- +DROP TABLE job_audit; +DROP TABLE outbox; +DROP TABLE attachment_jobs; +ALTER TABLE attachments DROP COLUMN status,DROP COLUMN error_code,DROP COLUMN pipeline_version,DROP COLUMN staging_key,DROP COLUMN variants,DROP COLUMN variant_keys; diff --git a/service/migrations/009_variant_manifest.sql b/service/migrations/009_variant_manifest.sql new file mode 100644 index 0000000..84413c8 --- /dev/null +++ b/service/migrations/009_variant_manifest.sql @@ -0,0 +1,8 @@ +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL, + variant text NOT NULL CHECK(variant IN ('original','thumbnail','crop','optimized')), + object_key text NOT NULL UNIQUE, sha256 text NOT NULL, size_bytes bigint NOT NULL, + PRIMARY KEY(attachment_id,pipeline_version,variant) +); +---- create above / drop below ---- +DROP TABLE attachment_variants; diff --git a/service/migrations/010_observability.sql b/service/migrations/010_observability.sql new file mode 100644 index 0000000..f6ac457 --- /dev/null +++ b/service/migrations/010_observability.sql @@ -0,0 +1,10 @@ +ALTER TABLE attachment_jobs ADD COLUMN traceparent text NOT NULL DEFAULT '', ADD COLUMN tracestate text NOT NULL DEFAULT ''; +CREATE TABLE job_slo ( + job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id) DEFERRABLE INITIALLY DEFERRED, + accepted_at timestamptz NOT NULL, deadline_at timestamptz NOT NULL, + ready_at timestamptz, outcome text CHECK(outcome IN ('good','bad')), evaluated_at timestamptz +); +CREATE INDEX job_slo_pending ON job_slo(deadline_at) WHERE outcome IS NULL; +---- create above / drop below ---- +DROP TABLE job_slo; +ALTER TABLE attachment_jobs DROP COLUMN traceparent, DROP COLUMN tracestate; diff --git a/service/migrations/011_terminal_slo.sql b/service/migrations/011_terminal_slo.sql new file mode 100644 index 0000000..d7f25d8 --- /dev/null +++ b/service/migrations/011_terminal_slo.sql @@ -0,0 +1,3 @@ +ALTER TABLE job_slo ADD COLUMN failed_at timestamptz; +---- create above / drop below ---- +ALTER TABLE job_slo DROP COLUMN failed_at; diff --git a/service/migrations/embed.go b/service/migrations/embed.go new file mode 100644 index 0000000..eb05421 --- /dev/null +++ b/service/migrations/embed.go @@ -0,0 +1,8 @@ +package migrations + +import ( + "embed" +) + +//go:embed *.sql +var Migrations embed.FS diff --git a/service/tests/api_test.go b/service/tests/api_test.go new file mode 100644 index 0000000..720cea8 --- /dev/null +++ b/service/tests/api_test.go @@ -0,0 +1,285 @@ +package tests + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + "io" + "net/http" + "net/url" + "os" + "strings" + "testing" + "time" +) + +type apiClient struct { + baseURL string + prefix string + tokens map[string]string +} + +type apiResponse struct { + status int + requestID string + instanceID string + body []byte +} + +func newAPI(t *testing.T) *apiClient { + t.Helper() + base := os.Getenv("TEST_BASE_URL") + if base == "" { + base = "http://localhost:18086" + } + return &apiClient{baseURL: strings.TrimRight(base, "/"), prefix: uuid.NewString()[:8] + "_", tokens: map[string]string{}} +} +func (api *apiClient) request(t *testing.T, method, path, userID, body string) apiResponse { + return api.requestKey(t, method, path, userID, body, uuid.NewString()) +} +func (api *apiClient) requestKey(t *testing.T, method, path, userID, body, key string) apiResponse { + t.Helper() + response, err := api.send(method, path, userID, body, key) + if err != nil { + t.Fatal(err) + } + return response +} +func (api *apiClient) send(method, path, userID, body, key string) (apiResponse, error) { + if path == "/api/v1/users" { + var fields map[string]any + if json.Unmarshal([]byte(body), &fields) == nil { + if name, ok := fields["username"].(string); ok { + fields["username"] = api.prefix + name + data, err := json.Marshal(fields) + if err != nil { + return apiResponse{}, err + } + body = string(data) + } + } + } + req, err := http.NewRequest(method, api.baseURL+path, strings.NewReader(body)) + if err != nil { + return apiResponse{}, err + } + if body != "" { + req.Header.Set("Content-Type", "application/json") + } + if userID != "" { + req.Header.Set("Authorization", "Bearer "+api.tokens[userID]) + } + if key != "" { + req.Header.Set("Idempotency-Key", key) + } + response, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) + if err != nil { + return apiResponse{}, fmt.Errorf("running lab5 stand required: %w", err) + } + defer response.Body.Close() + data, err := io.ReadAll(response.Body) + if err != nil { + return apiResponse{}, err + } + requestID, instanceID := response.Header.Get("X-Request-Id"), response.Header.Get("X-Instance-Id") + if requestID == "" || instanceID == "" { + return apiResponse{}, fmt.Errorf("%s %s: HTTP %d missing response identity", method, path, response.StatusCode) + } + return apiResponse{status: response.StatusCode, requestID: requestID, instanceID: instanceID, body: data}, nil +} + +func jsonString(t *testing.T, value any) string { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatalf("encode request: %v", err) + } + return string(data) +} + +func decodeOK(t *testing.T, resp apiResponse, wantStatus int, target any) { + t.Helper() + if resp.status != wantStatus { + t.Fatalf("status = %d, want %d; body: %s", resp.status, wantStatus, resp.body) + } + if err := json.Unmarshal(resp.body, target); err != nil { + t.Fatalf("decode response %s: %v", resp.body, err) + } +} + +func assertError(t *testing.T, resp apiResponse, wantStatus int, wantCode string) { + t.Helper() + if resp.status != wantStatus { + t.Fatalf("status = %d, want %d; body: %s", resp.status, wantStatus, resp.body) + } + var body struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` + } + if err := json.Unmarshal(resp.body, &body); err != nil { + t.Fatalf("decode error response %s: %v", resp.body, err) + } + if body.Error.Code != wantCode || body.Error.Message == "" || body.Error.RequestID != resp.requestID { + t.Fatalf("invalid error response: header request ID %q, body %s", resp.requestID, resp.body) + } +} + +func (api *apiClient) createUser(t *testing.T, username string) string { + t.Helper() + resp := api.request(t, http.MethodPost, "/api/v1/users", "", jsonString(t, map[string]string{ + "username": username, "display_name": username, "password": "Synthetic-password-123", + })) + var user struct { + ID string `json:"id"` + } + decodeOK(t, resp, http.StatusCreated, &user) + if user.ID == "" { + t.Fatal("created user has no ID") + } + req, _ := http.NewRequest(http.MethodPost, api.baseURL+"/api/v1/auth/sessions", nil) + req.SetBasicAuth(api.prefix+username, "Synthetic-password-123") + sessionResp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) + if err != nil { + t.Fatal(err) + } + defer sessionResp.Body.Close() + var session struct { + Token string `json:"token"` + } + if err = json.NewDecoder(sessionResp.Body).Decode(&session); err != nil || sessionResp.StatusCode != 201 || session.Token == "" { + t.Fatalf("login failed: status %d, error %v", sessionResp.StatusCode, err) + } + api.tokens[user.ID] = session.Token + return user.ID +} + +func (api *apiClient) createChat(t *testing.T, creatorID string, memberIDs ...string) string { + t.Helper() + resp := api.request(t, http.MethodPost, "/api/v1/chats", creatorID, jsonString(t, map[string]any{ + "title": "Test chat", "member_ids": memberIDs, + })) + var chat struct { + ID string `json:"id"` + } + decodeOK(t, resp, http.StatusCreated, &chat) + if chat.ID == "" { + t.Fatal("created chat has no ID") + } + return chat.ID +} + +func TestInvalidJSON(t *testing.T) { + api := newAPI(t) + for _, tc := range []struct { + name string + body string + }{ + {name: "malformed", body: `{"username":`}, + {name: "wrong type", body: `{"username":42,"display_name":"Alice"}`}, + {name: "unknown field", body: `{"username":"alice","display_name":"Alice","admin":true}`}, + } { + t.Run(tc.name, func(t *testing.T) { + resp := api.request(t, http.MethodPost, "/api/v1/users", "", tc.body) + assertError(t, resp, http.StatusBadRequest, "invalid_request") + }) + } +} + +func TestForeignChatIsHidden(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + bob := api.createUser(t, "bob") + eve := api.createUser(t, "eve") + chatID := api.createChat(t, alice, alice, bob) + + for _, tc := range []struct { + method string + path string + body string + }{ + {method: http.MethodGet, path: "/api/v1/chats/" + chatID}, + {method: http.MethodGet, path: "/api/v1/chats/" + chatID + "/messages"}, + {method: http.MethodPost, path: "/api/v1/chats/" + chatID + "/messages", body: `{"text":"secret"}`}, + } { + resp := api.request(t, tc.method, tc.path, eve, tc.body) + assertError(t, resp, http.StatusNotFound, "not_found") + } +} + +func TestEmptyMessage(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + chatID := api.createChat(t, alice, alice) + path := "/api/v1/chats/" + chatID + "/messages" + for _, body := range []string{`{"text":""}`, `{"text":" \t\n "}`} { + resp := api.request(t, http.MethodPost, path, alice, body) + assertError(t, resp, http.StatusBadRequest, "invalid_request") + } +} + +func TestDuplicateUsername(t *testing.T) { + api := newAPI(t) + api.createUser(t, "alice") + resp := api.request(t, http.MethodPost, "/api/v1/users", "", `{"username":"alice","display_name":"Another Alice","password":"Synthetic-password-123"}`) + assertError(t, resp, http.StatusConflict, "conflict") +} + +func TestMessagePagination(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + chatID := api.createChat(t, alice, alice) + wantIDs := make(map[string]bool) + for i := range 5 { + resp := api.request(t, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", alice, + jsonString(t, map[string]string{"text": fmt.Sprintf("message %d", i)})) + var message struct { + ID string `json:"id"` + } + decodeOK(t, resp, http.StatusCreated, &message) + if message.ID == "" || wantIDs[message.ID] { + t.Fatalf("invalid or duplicate created message ID %q", message.ID) + } + wantIDs[message.ID] = true + } + + seen := make(map[string]bool) + cursor := "" + pages := 0 + for { + path := "/api/v1/chats/" + chatID + "/messages?limit=2" + if cursor != "" { + path += "&cursor=" + url.QueryEscape(cursor) + } + resp := api.request(t, http.MethodGet, path, alice, "") + var page struct { + Items []struct { + ID string `json:"id"` + } `json:"items"` + NextCursor *string `json:"next_cursor"` + } + decodeOK(t, resp, http.StatusOK, &page) + pages++ + if pages > 3 || len(page.Items) == 0 || len(page.Items) > 2 { + t.Fatalf("unexpected page %d: %s", pages, resp.body) + } + for _, message := range page.Items { + if !wantIDs[message.ID] || seen[message.ID] { + t.Fatalf("unexpected or repeated message ID %q on page %d", message.ID, pages) + } + seen[message.ID] = true + } + if page.NextCursor == nil { + break + } + if *page.NextCursor == "" || *page.NextCursor == cursor { + t.Fatalf("invalid next cursor on page %d: %s", pages, resp.body) + } + cursor = *page.NextCursor + } + if pages != 3 || len(seen) != len(wantIDs) { + t.Fatalf("got %d pages and %d unique messages, want 3 pages and %d messages", pages, len(seen), len(wantIDs)) + } +} diff --git a/service/tests/attachments_test.go b/service/tests/attachments_test.go new file mode 100644 index 0000000..1a3e6bf --- /dev/null +++ b/service/tests/attachments_test.go @@ -0,0 +1,226 @@ +package tests + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "hash/crc32" + "image" + "image/jpeg" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" +) + +func (api *apiClient) upload(t *testing.T, chat, user, name, mime string, data []byte, chunked bool) apiResponse { + t.Helper() + req, e := http.NewRequest("POST", api.baseURL+"/api/v1/chats/"+chat+"/attachments?filename="+url.QueryEscape(name), bytes.NewReader(data)) + if e != nil { + t.Fatal(e) + } + if chunked { + req.ContentLength = -1 + } + req.Header.Set("Content-Type", mime) + if user != "" { + req.Header.Set("Authorization", "Bearer "+api.tokens[user]) + } + resp, e := (&http.Client{Timeout: 20 * time.Second}).Do(req) + if e != nil { + t.Fatal(e) + } + defer resp.Body.Close() + body, e := io.ReadAll(resp.Body) + if e != nil { + t.Fatal(e) + } + if resp.Header.Get("X-Instance-Id") == "" { + t.Fatal("missing instance id") + } + return apiResponse{status: resp.StatusCode, body: body, requestID: resp.Header.Get("X-Request-Id"), instanceID: resp.Header.Get("X-Instance-Id")} +} +func fixturePNG(t *testing.T) []byte { + t.Helper() + b, e := os.ReadFile("../../tests/fixtures/sample.png") + if e != nil { + t.Fatal(e) + } + return b +} +func TestAttachmentContractAndErrors(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + bob := api.createUser(t, "bob") + eve := api.createUser(t, "eve") + chat := api.createChat(t, alice, alice, bob) + other := api.createChat(t, alice, alice) + data := fixturePNG(t) + var att struct { + ID string `json:"id"` + Sha string `json:"sha256"` + Size int `json:"size_bytes"` + } + decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 202, &att) + hash := sha256.Sum256(data) + if att.Sha != hex.EncodeToString(hash[:]) || att.Size != len(data) { + t.Fatal("original metadata mismatch") + } + waitAttachmentReady(t, api, alice, att.ID) + content := "/api/v1/attachments/" + att.ID + "/content" + got := api.request(t, "GET", content, bob, "") + if got.status != 200 || !bytes.Equal(got.body, data) { + t.Fatal("download changed bytes") + } + for _, suffix := range []string{"", "/content"} { + assertError(t, api.request(t, "GET", "/api/v1/attachments/"+att.ID+suffix, eve, ""), 404, "not_found") + } + assertError(t, api.upload(t, chat, eve, "file.png", "image/png", data, false), 404, "not_found") + assertError(t, api.upload(t, chat, "", "file.png", "image/png", data, false), 401, "unauthorized") + for _, name := range []string{"", " ", "../a.png", "a/b.png", "a\\b.png", "bad\x00.png", "bad\n.png", strings.Repeat("я", 256)} { + assertError(t, api.upload(t, chat, alice, name, "image/png", data, false), 400, "invalid_request") + } + for _, tc := range []struct { + name, mime string + data []byte + }{ + {"fake", "image/png", []byte("not an image")}, + {"mime mismatch", "image/jpeg", data}, + {"wrong type", "application/octet-stream", data}, + {"truncated", "image/png", data[:80]}, + } { + t.Run(tc.name, func(t *testing.T) { + assertError(t, api.upload(t, chat, alice, "test.png", tc.mime, tc.data, false), 415, "unsupported_media_type") + }) + } + bomb := bytes.Clone(data) + binary.BigEndian.PutUint32(bomb[16:20], 40000001) + binary.BigEndian.PutUint32(bomb[20:24], 1) + binary.BigEndian.PutUint32(bomb[29:33], crc32.ChecksumIEEE(bomb[12:29])) + assertError(t, api.upload(t, chat, alice, "bomb.png", "image/png", bomb, false), 413, "payload_too_large") + // A syntactically valid ancillary PNG chunk reaches the exact byte boundary + // without allocating an enormous decoded image. + padding := make([]byte, 10485760-len(data)-12) + chunk := append([]byte("paDd"), padding...) + boundary := append([]byte{}, data[:len(data)-12]...) + boundary = binary.BigEndian.AppendUint32(boundary, uint32(len(padding))) + boundary = append(boundary, chunk...) + boundary = binary.BigEndian.AppendUint32(boundary, crc32.ChecksumIEEE(chunk)) + boundary = append(boundary, data[len(data)-12:]...) + var boundaryAtt map[string]any + decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 202, &boundaryAtt) + + oversized := make([]byte, 10485761) + copy(oversized, data) + for _, chunked := range []bool{false, true} { + assertError(t, api.upload(t, chat, alice, "large.png", "image/png", oversized, chunked), 413, "payload_too_large") + } + var jpg bytes.Buffer + if e := jpeg.Encode(&jpg, image.NewRGBA(image.Rect(0, 0, 32, 24)), nil); e != nil { + t.Fatal(e) + } + var jpegAtt map[string]any + decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 202, &jpegAtt) + imageOnly := `{"text":"","attachment_ids":["` + att.ID + `"]}` + var message map[string]any + key := uuid.NewString() + decodeOK(t, api.requestKey(t, "POST", "/api/v1/chats/"+chat+"/messages", bob, imageOnly, key), 201, &message) + var replay map[string]any + decodeOK(t, api.requestKey(t, "POST", "/api/v1/chats/"+chat+"/messages", bob, imageOnly, key), 201, &replay) + if jsonString(t, message) != jsonString(t, replay) { + t.Fatal("attachment retry changed message") + } + assertError(t, api.requestKey(t, "POST", "/api/v1/chats/"+chat+"/messages", bob, `{"text":"changed","attachment_ids":["`+att.ID+`"]}`, key), 409, "conflict") + assertError(t, api.request(t, "POST", "/api/v1/chats/"+other+"/messages", alice, imageOnly), 404, "not_found") + for _, body := range []string{`{"text":"","attachment_ids":["` + uuid.NewString() + `"]}`} { + assertError(t, api.request(t, "POST", "/api/v1/chats/"+chat+"/messages", alice, body), 404, "not_found") + } + for _, body := range []string{`{"text":""}`, `{"text":" "}`, `{"attachment_ids":["` + att.ID + `"]}`, `{"text":"ok","attachment_ids":null}`, `{"text":"ok","attachment_ids":"bad"}`, `{"text":"ok","attachment_ids":["` + att.ID + `","` + att.ID + `"]}`} { + assertError(t, api.request(t, "POST", "/api/v1/chats/"+chat+"/messages", alice, body), 400, "invalid_request") + } + if got := api.request(t, "GET", content+"?variant=thumbnail", bob, ""); got.status != 200 { + t.Fatal("thumbnail unavailable") + } +} +func TestConcurrentAttachmentMessage(t *testing.T) { + api := newAPI(t) + user := api.createUser(t, "alice") + chat := api.createChat(t, user, user) + var att struct { + ID string `json:"id"` + } + decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 202, &att) + key := uuid.NewString() + body := `{"text":"","attachment_ids":["` + att.ID + `"]}` + type result struct { + resp apiResponse + err error + } + ch := make(chan result, 16) + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { + resp, e := api.send("POST", "/api/v1/chats/"+chat+"/messages", user, body, key) + ch <- result{resp, e} + }) + } + wg.Wait() + close(ch) + seen := map[string]bool{} + var id string + for item := range ch { + if item.err != nil { + t.Fatal(item.err) + } + var msg struct { + ID string `json:"id"` + IDs []string `json:"attachment_ids"` + } + decodeOK(t, item.resp, 201, &msg) + if id != "" && id != msg.ID { + t.Fatal("duplicate attachment message") + } + id = msg.ID + seen[item.resp.instanceID] = true + if len(msg.IDs) != 1 || msg.IDs[0] != att.ID { + t.Fatal("lost attachment") + } + } + if len(seen) < 2 { + t.Fatal("did not exercise two replicas") + } + var page struct { + Items []json.RawMessage `json:"items"` + } + decodeOK(t, api.request(t, "GET", "/api/v1/chats/"+chat+"/messages", user, ""), 200, &page) + if len(page.Items) != 1 { + t.Fatal("duplicate history") + } +} + +func waitAttachmentReady(t *testing.T, api *apiClient, user, id string) { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + var att struct { + Status string `json:"status"` + } + decodeOK(t, api.request(t, "GET", "/api/v1/attachments/"+id, user, ""), 200, &att) + if att.Status == "ready" { + return + } + if att.Status == "failed" { + t.Fatal("attachment failed") + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("attachment readiness deadline") +} diff --git a/service/tests/concurrency_test.go b/service/tests/concurrency_test.go new file mode 100644 index 0000000..c48ec8c --- /dev/null +++ b/service/tests/concurrency_test.go @@ -0,0 +1,161 @@ +package tests + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + "io" + "net/http" + "net/url" + "strings" + "sync" + "testing" + "time" +) + +type createdMessage struct { + ID string `json:"id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` +} + +func postMessage(client *http.Client, endpoint, userID, text string) (createdMessage, error) { + body, err := json.Marshal(struct { + Text string `json:"text"` + }{Text: text}) + if err != nil { + return createdMessage{}, err + } + req, err := http.NewRequest(http.MethodPost, endpoint, strings.NewReader(string(body))) + if err != nil { + return createdMessage{}, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+userID) + req.Header.Set("Idempotency-Key", uuid.NewString()) + resp, err := client.Do(req) + if err != nil { + return createdMessage{}, err + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return createdMessage{}, err + } + if resp.StatusCode != http.StatusCreated { + return createdMessage{}, fmt.Errorf("POST status %d: %s", resp.StatusCode, data) + } + if resp.Header.Get("X-Request-Id") == "" { + return createdMessage{}, fmt.Errorf("POST response has no X-Request-Id") + } + var message createdMessage + if err := json.Unmarshal(data, &message); err != nil { + return createdMessage{}, fmt.Errorf("decode POST response: %w", err) + } + return message, nil +} + +func TestConcurrentMessages(t *testing.T) { + const clients = 10 + const messagesPerClient = 10 + const total = clients * messagesPerClient + + api := newAPI(t) + users := make([]string, clients) + for i := range users { + users[i] = api.createUser(t, fmt.Sprintf("user%02d", i)) + } + chatID := api.createChat(t, users[0], users...) + endpoint := api.baseURL + "/api/v1/chats/" + chatID + "/messages" + + type result struct { + message createdMessage + wantSenderID string + wantText string + err error + } + results := make(chan result, total) + start := make(chan struct{}) + var ready sync.WaitGroup + var workers sync.WaitGroup + ready.Add(clients) + workers.Add(clients) + for i, userID := range users { + go func(workerID int, userID string) { + defer workers.Done() + client := &http.Client{Timeout: 5 * time.Second} + defer client.CloseIdleConnections() + ready.Done() + <-start + for j := range messagesPerClient { + text := fmt.Sprintf("client-%02d-message-%02d", workerID, j) + message, err := postMessage(client, endpoint, api.tokens[userID], text) + results <- result{message: message, wantSenderID: userID, wantText: text, err: err} + } + }(i, userID) + } + ready.Wait() + close(start) + workers.Wait() + close(results) + + created := make(map[string]createdMessage, total) + count := 0 + for item := range results { + count++ + if item.err != nil { + t.Errorf("send %q: %v", item.wantText, item.err) + continue + } + if item.message.ID == "" || item.message.SenderID != item.wantSenderID || item.message.Text != item.wantText { + t.Errorf("unexpected response for %q: %+v", item.wantText, item.message) + continue + } + if _, duplicate := created[item.message.ID]; duplicate { + t.Errorf("duplicate message ID %q", item.message.ID) + continue + } + created[item.message.ID] = item.message + } + if count != total || len(created) != total { + t.Fatalf("sent %d requests, created %d unique messages; want %d", count, len(created), total) + } + + seen := make(map[string]bool, total) + cursor := "" + pages := 0 + for { + path := "/api/v1/chats/" + chatID + "/messages?limit=17" + if cursor != "" { + path += "&cursor=" + url.QueryEscape(cursor) + } + resp := api.request(t, http.MethodGet, path, users[0], "") + var page struct { + Items []createdMessage `json:"items"` + NextCursor *string `json:"next_cursor"` + } + decodeOK(t, resp, http.StatusOK, &page) + pages++ + if pages > total || len(page.Items) == 0 || len(page.Items) > 17 { + t.Fatalf("unexpected page %d: %s", pages, resp.body) + } + for _, message := range page.Items { + want, exists := created[message.ID] + if !exists || seen[message.ID] || message != want { + t.Fatalf("unexpected or repeated message on page %d: %+v", pages, message) + } + seen[message.ID] = true + } + if page.NextCursor == nil { + break + } + if *page.NextCursor == "" || *page.NextCursor == cursor { + t.Fatalf("invalid cursor on page %d: %s", pages, resp.body) + } + cursor = *page.NextCursor + } + if len(seen) != total { + t.Fatalf("history has %d unique messages across %d pages; want %d", len(seen), pages, total) + } + t.Logf("%d clients sent %d messages; history contains %d unique messages across %d pages", clients, total, len(seen), pages) +} diff --git a/service/tests/idempotency_test.go b/service/tests/idempotency_test.go new file mode 100644 index 0000000..9b317fd --- /dev/null +++ b/service/tests/idempotency_test.go @@ -0,0 +1,104 @@ +package tests + +import ( + "fmt" + "github.com/google/uuid" + "net/http" + "sync" + "testing" +) + +func TestIdempotencyContract(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + bob := api.createUser(t, "bob") + eve := api.createUser(t, "eve") + chat := api.createChat(t, alice, alice, bob) + other := api.createChat(t, alice, alice) + path := "/api/v1/chats/" + chat + "/messages" + for _, key := range []string{"", "bad key", "кириллица", string(make([]byte, 129))} { + // NUL cannot be sent by net/http: use an overlength ASCII key instead. + if len(key) == 129 { + key = fmt.Sprintf("%0129d", 1) + } + assertError(t, api.requestKey(t, "POST", path, alice, `{"text":"hello"}`, key), 400, "invalid_request") + } + key := uuid.NewString() + first := api.requestKey(t, "POST", path, alice, `{"text":"hello"}`, key) + var want map[string]any + decodeOK(t, first, 201, &want) + var got map[string]any + decodeOK(t, api.requestKey(t, "POST", path, alice, `{ "text" : "hello" }`, key), 201, &got) + if jsonString(t, want) != jsonString(t, got) { + t.Fatal("replay changed message") + } + assertError(t, api.requestKey(t, "POST", path, alice, `{"text":"changed"}`, key), 409, "conflict") + assertError(t, api.requestKey(t, "POST", path, eve, `{"text":"hello"}`, key), 404, "not_found") + for _, tc := range []struct{ user, path string }{{bob, path}, {alice, "/api/v1/chats/" + other + "/messages"}} { + var msg map[string]any + decodeOK(t, api.requestKey(t, "POST", tc.path, tc.user, `{"text":"hello"}`, key), 201, &msg) + if msg["id"] == want["id"] { + t.Fatal("scope not isolated") + } + } + resp := api.request(t, http.MethodDelete, "/api/v1/auth/sessions/current", alice, "") + if resp.status != 204 { + t.Fatal(resp.status) + } + assertError(t, api.requestKey(t, "POST", path, alice, `{"text":"hello"}`, key), 401, "unauthorized") +} +func TestConcurrentIdempotencyAcrossReplicas(t *testing.T) { + api := newAPI(t) + user := api.createUser(t, "alice") + chat := api.createChat(t, user, user) + path := "/api/v1/chats/" + chat + "/messages" + key := uuid.NewString() + const count = 32 + type result struct { + response apiResponse + err error + } + responses := make(chan result, count) + start := make(chan struct{}) + var wg sync.WaitGroup + for range count { + wg.Go(func() { + <-start + response, err := api.send("POST", path, user, `{"text":"one logical message"}`, key) + responses <- result{response: response, err: err} + }) + } + close(start) + wg.Wait() + close(responses) + instances := map[string]bool{} + var id string + for result := range responses { + if result.err != nil { + t.Fatal(result.err) + } + resp := result.response + var msg struct { + ID string `json:"id"` + } + decodeOK(t, resp, 201, &msg) + if id != "" && id != msg.ID { + t.Fatal("duplicate side effect") + } + id = msg.ID + instances[resp.instanceID] = true + } + if len(instances) < 2 { + t.Fatalf("need at least 2 real API replicas, observed %d", len(instances)) + } + var page struct { + Items []struct { + ID string `json:"id"` + } `json:"items"` + } + decodeOK(t, api.request(t, "GET", path, user, ""), 200, &page) + if len(page.Items) != 1 || page.Items[0].ID != id { + t.Fatalf("history contains %d messages", len(page.Items)) + } + t.Logf("%d requests across %d replicas created exactly one message", count, len(instances)) +} diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 0000000..ece8e3e --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,10 @@ +# Перенесённые проверки pipeline + +Существующие проверки перенесены из собственной lab5 и используют только изолированный Compose-проект lab6, URL http://localhost:18086 и его volumes. Новые unit/integration-тесты для ЛР №6 не добавлялись. Новые воспроизводимые сценарии наблюдаемости находятся в `scripts/`. + +- `make test-images`: форматы, геометрия, EXIF, alpha, checksums, ACL и отрицательные входы. +- `make test-staging`: защита активного source, cleanup, terminal error, retention, restore/replay. +- `make test-broker-persistence`: замена контейнера брокера с сохранённым volume при остановленном потребителе. +- `python3 student-tests/pipeline.py --repeats 1 --output evidence/pipeline.json`: 13 сценариев crash/retry/lease/version/idempotency. Профиль ограничивает job timeout/lease; в finally восстанавливается стандартная конфигурация. + +Все изменяющие состояние проверки запускать последовательно на синтетическом стенде. Нельзя выполнять их параллельно с исследованием SLO. Токены не печатаются и не сохраняются. diff --git a/student-tests/broker_persistence.py b/student-tests/broker_persistence.py new file mode 100644 index 0000000..ae1209e --- /dev/null +++ b/student-tests/broker_persistence.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Prove a confirmed queued message survives broker container replacement, not only restart.""" +from pathlib import Path +import json +import subprocess +import sys +import time +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check + +def compose(*args): + r=subprocess.run([str(ROOT/'scripts/compose.sh'),*args],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=150) + check(r.returncode==0,'broker persistence infrastructure command failed: '+args[0]);return r.stdout.strip() + +def queued(): + r=subprocess.run([str(ROOT/'scripts/compose.sh'),'exec','-T','broker','rabbitmqctl','-q','list_queues','name','messages'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=30) + if r.returncode:return 0 + out=r.stdout + for line in out.splitlines(): + fields=line.split() + if len(fields)==2 and fields[0]=='attachments.v1':return int(fields[1]) + return 0 + +def main(): + c=Client('http://localhost:18086');alice,bob,eve,chat=c.bootstrap() + try: + compose('stop','worker');att,data=c.attachment(alice,chat) + deadline=time.monotonic()+20 + while queued()<1 and time.monotonic()=1,'message not durably queued before replacement') + published=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-c',"SELECT bool_and(status='published') FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%att['id']) + check(published=='t','outbox not marked published before replacement') + compose('stop','dispatcher') + old=compose('ps','-q','broker') + compose('up','-d','--no-deps','--force-recreate','--wait','--wait-timeout','120','broker') + new=compose('ps','-q','broker');check(new!=old,'broker container was not replaced') + deadline=time.monotonic()+30 + while queued()<1 and time.monotonic()=1,'confirmed queue message lost after replacement') + compose('start','worker');c.wait_ready(bob,att) + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'original changed after broker replacement') + result={'result':'PASS','attachment_id':att['id'],'broker_hostname':'broker','old_container_id':old,'new_container_id':new,'message_present_before_and_after':True,'outbox_published_before_replacement':True,'dispatcher_stopped_during_replacement':True} + (ROOT/'evidence/broker-persistence.json').write_text(json.dumps(result,indent=2)+'\n') + print('PASS: confirmed message survived broker container replacement with the same volume and stable node identity') + finally:compose('start','worker','dispatcher') +if __name__=='__main__':main() diff --git a/student-tests/images.py b/student-tests/images.py new file mode 100644 index 0000000..4ef96fd --- /dev/null +++ b/student-tests/images.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Image contract checks using synthetic fixtures and real decoding in ImageMagick.""" +import hashlib +import json +from pathlib import Path +import struct +import subprocess +import sys +import urllib.parse +import zlib +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check +from image_probe import jpeg_size + +def magick(args,data=None): + r=subprocess.run([str(ROOT/'scripts/compose.sh'),'run','--rm','--no-deps','-T','--entrypoint','magick','operator']+args,input=data,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=30) + check(r.returncode==0,'fixture decoder failed') + return r.stdout + +def png(w,h,rgba): + def chunk(t,b):return struct.pack('>I',len(b))+t+b+struct.pack('>I',zlib.crc32(t+b)) + return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',zlib.compress(b''.join(b'\0'+b''.join(bytes(rgba(x,y)) for x in range(w)) for y in range(h)) if callable(rgba) else (b'\0'+bytes(rgba)*w)*h))+chunk(b'IEND',b'') + +def main(): + c=Client('http://localhost:18086');alice,bob,eve,chat=c.bootstrap();results=[] + jpeg=magick(['png:-','jpeg:-'],png(300,150,(255,0,0,255))) + # EXIF TIFF orientation 6: display rotates the stored 300x150 pixels clockwise. + tiff=b'II'+struct.pack('H',len(exif)+2)+exif+jpeg[2:] + for name,data,mime,dimensions in [('small',png(16,8,(255,0,0,255)),'image/png',(16,8)),('floor',png(333,100,(255,0,0,255)),'image/png',(256,76)),('alpha',png(40,20,(0,0,0,0)),'image/png',(40,20)),('orientation',oriented,'image/jpeg',(128,256)),('thin',png(1,300,(0,0,0,255)),'image/png',(1,256)),('center_crop',png(300,100,lambda x,y:(255,0,0,255) if 100<=x<200 else (0,0,255,255)),'image/png',(256,85))]: + att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+name,202,actor=alice,raw=data,headers={'Content-Type':mime},schema='Attachment')['json'];ready=c.wait_ready(bob,att) + original=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes'];check(original==data,'original changed') + for v in ready['variants']: + b=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+v['name'],actor=bob)['bytes'];check(hashlib.sha256(b).hexdigest()==v['sha256'],'checksum mismatch') + actual=tuple(map(int,magick(['identify','-format','%w %h','-'],b).split())) + check(actual==(v['width'],v['height']),'actual dimensions differ') + if v['name']=='thumbnail': + check(jpeg_size(b)==dimensions,'thumbnail geometry differs') + if name=='alpha':check(magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b).strip() in (b'gray(255)',b'srgb(255,255,255)'),'alpha was not composited on white') + if v['name']=='crop': + check(actual==(128,128),'crop geometry differs') + if name=='center_crop': + import re + color=tuple(map(int,re.findall(rb'\d+',magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b)))) + check(len(color)==3 and color[0]>=250 and color[1]<=5 and color[2]<=5,'crop is not centered') + check(b'Exif\0\0' not in b,'EXIF retained in derivative') + c.call('GET','/api/v1/attachments/'+att['id']+'/content',404,actor=eve) + results.append({'case':name,'result':'PASS','thumbnail':dimensions}) + for data,mime,status in [(b'not image','image/png',415),(png(2,2,(0,0,0,255)),'image/jpeg',415),(png(2,2,(0,0,0,255))[:40],'image/png',415),(b'x'*10485761,'image/png',413)]: + c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=bad',status,actor=alice,raw=data,headers={'Content-Type':mime}) + (ROOT/'evidence/images.json').write_text(json.dumps({'results':results,'negative_cases':4},indent=2)+'\n');print('PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs') +if __name__=='__main__':main() diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100644 index 0000000..2b0a515 --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Repeatable lab5 experiment on the local synthetic Compose stand. Never prints tokens.""" +import argparse +import concurrent.futures +import datetime +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, Failure, check +from image_probe import jpeg_size +FAULTS=ROOT/'.lab6-faults' +COMPOSE=[str(ROOT/'scripts/compose.sh'),'-f',str(ROOT/'compose.yaml'),'-f',str(ROOT/'compose.faults.yaml')] +RESULTS=[] +SELECTED=set() + +def compose(*args, allow_failure=False): + p=subprocess.run(COMPOSE+list(args),cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=180) + if p.returncode and not allow_failure: + # Infrastructure logs may contain settings; report only action and exit status. + raise Failure('Compose '+args[0]+' failed, exit '+str(p.returncode)) + return p + +def sql(query): + return compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query).stdout.strip() + +def uid(value): + return str(uuid.UUID(value)) + +def job(att,version=1): + return sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s' AND pipeline_version=%d"%(uid(att['id']),version)) + +def operator(action,id=None,version=None, allow_failure=False): + args=['run','--rm','--no-deps','-T','operator','jobs',action] + if id: args+=['--id',uid(id)] + if version:args+=['--pipeline-version',str(version)] + return compose(*args,allow_failure=allow_failure) + +def wait(predicate,timeout=65): + deadline=time.monotonic()+timeout + while time.monotonic()=2,'lease not recovered') + operator('reprocess',att['id'],2);second=ready(c,bob,att,data) + check(first['variants']!=second['variants'],'pipeline versions are indistinguishable') + operator('redeliver',jid);marker.unlink();time.sleep(1) + check(metadata(c,bob,att)==second,'stale worker/job overwrote version 2') + check(sql("SELECT pipeline_version FROM attachments WHERE id='%s'"%uid(att['id']))=='2','version rolled back') + ready(c,bob,att,data) + return {'attachment_id':att['id'],'old_job_id':jid,'pipeline_version':2,'manifest_rows':8} + record('lease_reorder_and_stale_version',run,version_fence) + def retries(): + att,data,jid=upload_stopped();marker=directive('before_objects',jid,'temporary');compose('start','worker');failed=state(c,bob,att,'failed',65) + check(failed['error_code']=='retry_exhausted','wrong terminal code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='5','attempt limit not honored') + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + marker.unlink();operator('replay',jid);ready(c,bob,att,data);check(operator('replay',jid,allow_failure=True).returncode!=0,'successful job replay allowed') + return {'job_id':jid,'failed_attempt':5,'replayed_attempt':6} + record('bounded_retry_and_explicit_replay',run,retries) + def permanent(): + att,data,jid=upload_stopped();marker=directive('before_objects',jid,'permanent');compose('start','worker');failed=state(c,bob,att,'failed');check(failed['error_code']=='invalid_image','wrong permanent code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='1','permanent error was retried');marker.unlink();operator('replay',jid);ready(c,bob,att,data);return {'job_id':jid} + record('permanent_error',run,permanent) + for point in ('after_staging','after_upload_commit'): + def api_crash(point=point): + if point=='after_upload_commit':compose('stop','dispatcher') + marker=directive(point,'all','crash') + try:c.attachment(alice,chat) + except Failure:pass + aid=hit(marker);att={'id':aid} + if point=='after_staging':check(sql("SELECT count(*) FROM attachments WHERE id='%s'"%uid(aid))=='0','precommit row exists') + else: + check(sql("SELECT status FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%uid(aid))=='pending','COMMIT/publish window not reached') + compose('start','dispatcher');ready(c,bob,att,(ROOT/'tests/fixtures/sample.png').read_bytes()) + wait(lambda:health(c));return {'attachment_id':aid,'accepted_commit':point=='after_upload_commit'} + record('api_crash_'+point,run,api_crash) + payload={'date_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),'working_tree':'implementation changes uncommitted','repeats':args.repeats,'worker':{'concurrency':2,'prefetch':2,'timeout_seconds':5,'lease_seconds':8,'memory':'1GiB','cpus':1},'results':RESULTS} + check(bool(RESULTS),'no scenarios selected') + payload['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab6','--format','{{.Id}}'],text=True).strip() + (ROOT/args.output).write_text(json.dumps(payload,ensure_ascii=False,indent=2)+'\n') + finally: + for f in FAULTS.iterdir():f.unlink() + compose('stop','worker','dispatcher',allow_failure=True) + # Restore standard configuration, disabling fault injection and short experiment leases. + subprocess.run([str(ROOT/'scripts/compose.sh'),'up','-d','--wait','--wait-timeout','120'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=180) + +def health(c): + try:c.call('GET','/health/ready',schema='Health');return True + except Failure:return False + +if __name__=='__main__':main() diff --git a/student-tests/storage.py b/student-tests/storage.py new file mode 100644 index 0000000..0e583dd --- /dev/null +++ b/student-tests/storage.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Staging protection, retirement, retention and explicit restoration on synthetic data.""" +import json +from pathlib import Path +import sys +import time +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check +BASE=[str(ROOT/'scripts/compose.sh')] +import subprocess + +def command(*args,fail=False): + r=subprocess.run(BASE+list(args),stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=120) + if not fail:check(r.returncode==0,'storage scenario infrastructure command failed: '+args[0]) + return r + +def sql(q):return command('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',q).stdout.strip() +def op(*args,fail=False):return command('run','--rm','--no-deps','-T','operator',*args,fail=fail) +def wait(fn): + deadline=time.monotonic()+20 + while time.monotonic()0 and exists(key),'reconciliation removed active source') + results.append({'scenario':'queued_staging_protected','result':'PASS','attachment_id':att['id']}) + command('start','worker');c.wait_ready(bob,att) + wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%att['id'])=='') + check(not exists(key),'ready source staging not retired') + results.append({'scenario':'staging_deleted_after_ready','result':'PASS','attachment_id':att['id']}) + command('stop','worker') + failed,data=c.attachment(alice,chat);key='staging/'+failed['id'];jid=sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s'"%failed['id']) + op('storage','remove-object','--id',failed['id']);command('start','worker') + def is_failed():return c.call('GET','/api/v1/attachments/'+failed['id'],actor=bob,schema='Attachment')['json']['status']=='failed' + wait(is_failed) + check(sql("SELECT error_code FROM attachment_jobs WHERE id='%s'"%jid)=='source_missing','missing source not permanent') + check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted missing source') + op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png') + check(exists(key),'restored source missing') + # Advance only the synthetic record's age; do not wait a real day or change production retention. + sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '25 hours' WHERE id='%s'"%failed['id']) + wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%failed['id'])=='') + check(not exists(key),'expired failed source not retired') + check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted retired source') + results.append({'scenario':'failed_retention_and_missing_source_replay','result':'PASS','age_simulated_hours':25}) + op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png');op('jobs','replay','--id',jid);c.wait_ready(bob,failed) + check(c.call('GET','/api/v1/attachments/'+failed['id']+'/content',actor=bob)['bytes']==data,'restored original changed') + results.append({'scenario':'verified_restore_then_replay','result':'PASS','job_id':jid}) + (ROOT/'evidence/storage.json').write_text(json.dumps({'results':results},indent=2)+'\n') + print('PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified') + finally:command('start','worker') +if __name__=='__main__':main()