From fa3a69d9b4267939fae8db422dff31cd079725f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 19:18:27 +0300 Subject: [PATCH] docs: record verified failover and rolling replacement evidence --- REPORT.md | 107 +- evidence/cluster-first.json | 4654 +++++++++++++++++++++++++++++++++++ evidence/http-tests.json | 15 + research/run.py | 19 + 4 files changed, 4771 insertions(+), 24 deletions(-) create mode 100644 evidence/cluster-first.json create mode 100644 evidence/http-tests.json diff --git a/REPORT.md b/REPORT.md index c2679005..ad9561d0 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,96 @@ -# Отчёт — лабораторная 3 +# ЛР 3 — балансировка и отказ API-реплики -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Автор: **Маслов, БВТ2301**. Цель: **5, трек 1 — цена масштабирования**. +Площадка сдачи: Gitea, личная ветка `solution/bvt2301-maslov`. -## Запуск из чистого clone +## Запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker с Compose v2+ и Python 3.14 с uv; публичные преподавательские проверки работают также на Python 3.9+. Все команды выполняются в корне этого репозитория; соседние лабораторные не нужны. -## Архитектура и решения +```sh +uv sync --locked +make bootstrap +VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait +make check validate PYTHON=.venv/bin/python +NO_PROXY=localhost,127.0.0.1 make test BASE_URL=http://127.0.0.1:8083 +make test-student BASE_URL=http://127.0.0.1:8083 +make verify +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Bootstrap создаёт независимые случайные credentials в `secrets/` и `.env`, существующие файлы сохраняет. Каталог secrets имеет права 0700; файлы читаются непривилегированным UID10001 через read-only mounts. Секреты не находятся в Git или образе. Начальная база создаётся образом PostgreSQL; отдельный одноразовый `migrate` применяет SQL под advisory lock. API стартуют после успешной миграции. Изменённая применённая миграция отклоняется по SHA-256. -## Доказательства критериев +Основной адрес — `http://127.0.0.1:8083`. Порт можно изменить через `PUBLIC_PORT`; API и хранилища host ports не публикуют. В `.env.example` `API_REPLICAS` задаёт Compose scale, `API_CPUS` и `API_MEMORY` — квоты каждой API, `DB_POOL_SIZE` — максимум соединений каждого процесса. `SESSION_STORE=postgres` описывает выбранную архитектуру; переключателя на локальные сессии нет. Срок идемпотентности фиксирован контрактом в 86400 секунд, его уменьшение конфигурацией не поддерживается. `APP_ENV` — поясняющее поле skeleton, код его не использует. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +```sh +# Одна выбранная реплика остаётся остановленной: +NO_PROXY=localhost,127.0.0.1 make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8083 +# Явно вернуть требуемое число реплик: +docker compose up -d --no-build --wait +# Перезапуск API, PostgreSQL, Valkey и LB с сохранением volumes: +NO_PROXY=localhost,127.0.0.1 make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8083 +# Поочерёдно заменить все API, добавляя готовую реплику перед остановкой старой: +make rolling-update +# Безопасная остановка с сохранением данных: +docker compose stop +# Удалить контейнеры и сеть, сохранив данные: +make down +# Только намеренный сброс учебных данных ЭТОГО проекта: +docker compose down --volumes +``` -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Тесты создают синтетических пользователей и сообщения. Нагрузочные и аварийные сценарии запускаются явно. Для нового измерения выбирать новое имя output, сохраняя предыдущие результаты. -## Эксперименты и отказы +## Архитектура -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +```mermaid +flowchart LR + C[Клиент] --> LB[HAProxy] + LB --> A[API процесс 1] + LB --> B[API процесс 2] + A --> PG[(PostgreSQL)] + B --> PG + A --> V[(Valkey: disposable User cache)] + B --> V +``` -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Перенесена собственная реализация ЛР2: строгая валидация UTF-8 JSON и типов, ACL, keyset pagination `(created_at,id)`, Argon2id, случайные Bearer-сессии с абсолютным expiry, PostgreSQL и ограниченный по времени Valkey-кэш карточек User. Преподавательские README, COURSE, contracts и tests сохранены. Пароль не сохраняется открытым; для сессии хранится SHA-256 случайного 256-битного токена. Каждый защищённый запрос сначала проверяет действительность сессии в PostgreSQL. Кэш не может восстановить отозванную сессию или разрешить доступ при отказе БД. -## Ограничения и следующие шаги +Новые части: `X-Instance-Id` — UUID alias, генерируемый один раз внутри процесса API; HAProxy его не подменяет. Round robin применяется без sticky sessions. Одна сессия обслуживается любым процессом. Порты API доступны только внутри сети Compose; `container_name` не используется. Один uvicorn worker на контейнер, масштабирование — числом контейнеров. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +HAProxy 3.2.1 закреплён digest, использует Docker DNS 127.0.0.11 и server-template до 8 адресов. DNS hold valid/obsolete/nx — 1 с. HTTP readiness проверяет PostgreSQL каждые 1 с, fall2/rise1, fastinter250ms. Номинальный бюджет исключения отказавшего адреса — около 2 с плюс длительность зависшей readiness; connect timeout1с, HTTP server timeout8с. PostgreSQL-клиент имеет checkout deadline4с, pool wait1с, connect2с, statement2.5с и lock1с. Поэтому в реальном отказе возможны короткие ошибки; время подтверждается экспериментом. HAProxy не повторяет POST автоматически (`retries 0`). + +`/health/live` проверяет процесс; `/health/ready` — доступность обязательного синхронного PG-пути. Valkey необязателен для корректности и при runtime-отказе обходится с circuit backoff. Uvicorn завершает принятые запросы с бюджетом5с; контейнеру выделено10с. До импорта uvicorn действует ранний обработчик SIGTERM. Поочерёдная замена сначала добавляет готовую реплику, ждёт проверки и DNS, затем отправляет SIGTERM одной старой. Это проверка замены процессов одной версии; совместимость разных версий приложения отдельно не заявляется. Миграция002 — только расширение схемы, старый код ЛР2 не теряет свои таблицы. + +Ресурсы обычного стенда: API2×0.5CPU/384MiB, PostgreSQL1CPU/512MiB, Valkey0.25CPU/128MiB, LB0.5CPU/128MiB. API работают под UID10001, root filesystem read-only, tmpfs16MiB, capabilities удалены. Python3.14.7 и uv0.13.7 закреплены digest; зависимости — `uv.lock`. + +## Атомарность повторной отправки + +1. API проверяет Bearer и членство в чате, затем валидирует body и ASCII ключ `[A-Za-z0-9._:-]{1,128}`. +2. Область — пользователь, POST, нормализованный UUID чата и ключ. Другие чаты и пользователи независимы. В ЛР3 body содержит только text; схема вложений появляется в ЛР4. +3. В одной PG-транзакции берётся `pg_advisory_xact_lock` по первым64битам SHA-256 области. Коллизия lock лишь дополнительно сериализует независимые запросы: окончательное сравнение выполняется по полному составному PK. +4. Если результат моложе24ч, сравнивается SHA-256 канонического JSON. Другой text даёт409; тот же —201 с прежним Message, включая id и created_at. Порядок JSON-полей не влияет на hash. +5. Для нового/истёкшего ключа в той же транзакции вставляется Message и результат идемпотентности с expiry по clock_timestamp PostgreSQL. COMMIT подтверждает оба изменения; исключение откатывает оба. + +При неизвестном исходе POST клиент сохраняет исходные key/body и повторяет их. Повтор заново проверяет сессию/ACL. Ошибки5xx не сохраняются как успешный replay result. Записи истёкших ключей могут оставаться в таблице до обслуживания; expiry проверяется независимо от физического удаления. Контракт не обещает дедупликацию после24ч. Создание чата, регистрации и входа этим механизмом не защищены. + +## Проверки и исследование + +Проверки выполняются последовательно относительно изменений одного стенда. Первый собственный прогон был испорчен ошибочной пересборкой всех API/LB во время тестов: 503 и connection refused; этот прогон не засчитан. Повтор на готовом стенде: **87passed, 279.66с**; детали в `evidence/http-tests.json`. + +Результаты аварийных сценариев, матрицы масштабирования и чистого clone добавляются после их выполнения. Для уровня5 предусмотрены фиксированная суммарная квота API1CPU/1024MiB и серия с ростом ресурсов0.5CPU/256MiB на реплику; 1/2/4 процесса, C1/8/32, два повтора,2с прогрева и6с измерения. Смешанный closed-loop профиль:80% чтений истории limit20 и20% отправок с уникальными ключами. Перед каждой ячейкой в отдельном исследовательском проекте очищается только его новая БД и создаётся100 сообщений. Отдельно сравнивается пул1/10 при N2/C32 в трёх чередующихся повторах. Пока измерений нет, выигрыш не заявляется. + +## Вопросы защиты + +**Чем отказ процесса отличается от отказа узла?** Остановка одной API оставляет другие процессы, LB и общую БД живыми. Потеря этого Docker-host одновременно лишает нас всех контейнеров и локальных volumes. Этот стенд проверяет отказ процесса, многосерверной HA здесь нет. + +**Почему depends_on/healthcheck не являются балансировщиком?** depends_on определяет порядок запуска и начальную готовность. Healthcheck помечает состояние контейнера. Маршрутизацию клиентских запросов, исключение backend и DNS выполняет отдельный HAProxy. Его конфигурация нужна и после старта. + +**Какие POST повторять после timeout и что защищает ключ?** POST сообщения можно повторить с теми же key/body в пределах24ч: запись Message и результата общая и атомарная. Ключ не повторяет данные на клиенте и не защищает другой endpoint; registration/chat/login при неизвестном результате требуют отдельного решения. Timeout не означает гарантированный rollback. + +**Что сломается при локальных сессиях?** После round robin другая API не узнает токен, появятся случайные401; остановка владельца лишит сессии доступа. Sticky session скрыла бы часть ошибок до первого отказа. Общая PG-сессия сохраняет identity и абсолютный expiry через любую API и restart. + +## Границы и источники + +Один LB, PostgreSQL, Valkey и Docker-host; потеря БД/host не маскируется ростом API. TLS, репликация PostgreSQL, backup/PITR и промышленное наблюдение здесь не добавлены. HTTP доступен только на loopback, данные синтетические. Closed-loop имеет coordinated omission; короткий локальный прогон не задаёт production SLA. Пул — на процесс: рост реплик увеличивает возможное суммарное число PG-соединений. + +Источники: [HAProxy3.2 configuration](https://docs.haproxy.org/3.2/configuration.html), [PG18 explicit locks](https://www.postgresql.org/docs/18/explicit-locking.html), [Compose services](https://docs.docker.com/reference/compose-file/services/). Библиотеки перечислены в `pyproject.toml`, точные версии — `uv.lock`. Помощь ИИ использована при разборе требований, подготовке реализации и проверок; результаты подтверждаются сохранёнными запусками. diff --git a/evidence/cluster-first.json b/evidence/cluster-first.json new file mode 100644 index 00000000..c094c2a5 --- /dev/null +++ b/evidence/cluster-first.json @@ -0,0 +1,4654 @@ +{ + "date": "2026-11-15T16:14:56.143668+00:00", + "source_commit": "b14fb9a162d8012e83699cd8facdab178587d04b", + "complete": true, + "before": [ + { + "role": "api", + "container": "69839f9f1529", + "pid": 9378, + "started_at": "2026-11-15T16:07:47.8366298Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 402653184, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "role": "api", + "container": "2f77c0676d92", + "pid": 9434, + "started_at": "2026-11-15T16:07:47.950243717Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 402653184, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "role": "cache", + "container": "e44e44c0b312", + "pid": 856, + "started_at": "2026-11-15T16:04:08.466599796Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "lb", + "container": "4b5bda8bb42d", + "pid": 9874, + "started_at": "2026-11-15T16:07:56.068812346Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8083" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "f6c1860d783c", + "pid": 918, + "started_at": "2026-11-15T16:04:08.652994337Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + } + ], + "parallel_replay": { + "requests": 30, + "messages": 1, + "instances": [ + "d43818c500ec49a8bfd41bfa6a384231", + "e4ff9d5db09d4bfc8ece63cbf0c7985c" + ] + }, + "atomic_rollback_and_5xx_retry": true, + "expired_key_can_be_reused": true, + "failovers": [ + { + "confirmed_writes": 221, + "successful_reads": 215, + "errors": 7, + "successful_rps": 28.91, + "first_success_after_action_seconds": 4.191, + "action": { + "finished": 4.109 + }, + "observations": [ + { + "at": 0.007, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.06, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.117, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.173, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.231, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.307, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.363, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.427, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.488, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.546, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.609, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.665, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.725, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.781, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.839, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.9, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 0.959, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.02, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.08, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.143, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.214, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.272, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.337, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.4, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.466, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.528, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.584, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.643, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.698, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.76, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.86, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 1.963, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 2.063, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 2.163, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 2.267, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 2.332, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 2.389, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 2.502, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 2.593, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 2.699, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 2.797, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 2.901, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 2.991, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 3.102, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 3.2, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + }, + { + "at": 3.346, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 3.445, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 3.551, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 3.603, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 3.655, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 3.704, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 3.753, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 3.813, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 3.869, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 3.929, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.0, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.073, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.191, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.266, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.33, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.394, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.456, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.523, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.585, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.644, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.708, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.769, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.83, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.887, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 4.946, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.005, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.066, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.128, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.187, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.246, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.308, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.374, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.446, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.504, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.568, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.64, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.705, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.772, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.84, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.906, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 5.97, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.036, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.102, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.168, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.231, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.299, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.366, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.43, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.497, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.564, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.635, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.694, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.753, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.814, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.881, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 6.939, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.003, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.063, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.133, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.209, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.281, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.35, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.41, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.468, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.525, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.588, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.647, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.709, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.771, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.836, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.903, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 7.967, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.035, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.104, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.167, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.231, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.298, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.366, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.436, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.506, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.576, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.646, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.722, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.787, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.846, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.905, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 8.962, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.018, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.079, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.145, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.212, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.271, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.331, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.395, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.462, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.529, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.595, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.665, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.733, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.801, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.871, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 9.938, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.004, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.071, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.139, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.2, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.269, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.339, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.405, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.472, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.546, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.615, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.685, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.751, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.82, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.879, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.937, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 10.998, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.06, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.119, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.181, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.252, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.319, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.388, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.454, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.519, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.586, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.662, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.729, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.792, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.862, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 11.933, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.004, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.074, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.141, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.207, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.264, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.333, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.406, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.477, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.548, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.613, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.68, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.748, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.816, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.885, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 12.945, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.007, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.068, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.126, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.184, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.247, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.306, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.367, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.432, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.498, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.564, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.631, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.746, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.849, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 13.942, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.047, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.148, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.249, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.31, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.371, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.427, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.488, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.548, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.613, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.675, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.738, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.809, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.876, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 14.944, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 15.003, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "69839f9f1529", + "pid": 9378, + "started_at": "2026-11-15T16:07:47.8366298Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 402653184, + "instance": "e4ff9d5db09d4bfc8ece63cbf0c7985c" + } + }, + { + "confirmed_writes": 211, + "successful_reads": 203, + "errors": 9, + "successful_rps": 27.45, + "first_success_after_action_seconds": 3.643, + "action": { + "finished": 2.644 + }, + "observations": [ + { + "at": 0.012, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.068, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.13, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.187, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.249, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.309, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.367, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.425, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.485, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.546, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.606, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.661, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.719, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.776, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.83, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.898, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 0.995, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 1.101, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 1.2, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 1.296, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 1.393, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 1.471, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 1.575, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 1.671, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 1.774, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 1.876, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 1.977, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 2.075, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 2.132, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + }, + { + "at": 2.195, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.249, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.302, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.357, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.413, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.466, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.522, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.577, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.643, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.709, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.775, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.837, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.902, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.971, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.037, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.096, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.157, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.216, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.275, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.335, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.391, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.445, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.501, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.562, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.626, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.685, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.751, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.819, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.894, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.965, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.029, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.094, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.159, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.218, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.282, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.351, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.417, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.485, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.549, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.616, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.684, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.747, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.811, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.876, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.941, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.011, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.083, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.147, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.207, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.266, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.325, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.387, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.452, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.511, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.567, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.623, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.681, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.74, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.802, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.867, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.942, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.007, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.07, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.139, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.208, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.273, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.34, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.407, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.474, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.538, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.604, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.669, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.736, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.806, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.877, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.945, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.015, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.086, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.149, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.213, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.273, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.337, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.401, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.462, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.521, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.578, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.64, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.703, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.767, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.834, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.902, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.974, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.041, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.107, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.176, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.242, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.348, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.453, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.548, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.646, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.747, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.85, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.907, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.967, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.025, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.086, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.144, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.206, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.262, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.322, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.381, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.444, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.503, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.561, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.621, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.68, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.74, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.804, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.867, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.935, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.008, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.074, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.142, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.208, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.277, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.346, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.416, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.48, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.547, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.614, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.682, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.749, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.815, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.883, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.951, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.019, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.092, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.157, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.225, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.284, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.345, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.405, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.466, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.532, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.594, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.65, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.705, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.762, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.822, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.881, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.947, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.021, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.087, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.154, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.219, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.289, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.353, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.422, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.494, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.563, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.632, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.699, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.763, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.833, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.899, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.969, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.041, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.108, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.173, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.242, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.304, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.363, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.428, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.492, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.562, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.629, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.685, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.745, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.805, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.863, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.922, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 14.986, + "instance": "adb3bf3bda284356bfe1328423b66363" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "2f77c0676d92", + "pid": 9434, + "started_at": "2026-11-15T16:07:47.950243717Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 402653184, + "instance": "d43818c500ec49a8bfd41bfa6a384231" + } + } + ], + "rolling": { + "confirmed_writes": 655, + "successful_reads": 655, + "errors": 36, + "successful_rps": 29.02, + "first_success_after_action_seconds": 32.943, + "action": { + "finished": 32.934 + }, + "observations": [ + { + "at": 0.015, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 0.075, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 0.133, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 0.188, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 0.249, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 0.311, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 0.399, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 0.493, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 0.599, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 0.706, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 0.797, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 0.892, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 0.949, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.01, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.064, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.116, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.172, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.231, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.297, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.368, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.44, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.502, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.56, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.621, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.679, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.734, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.789, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.853, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.908, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 1.966, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.021, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.076, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.133, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.193, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.252, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.31, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.368, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.429, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.486, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.547, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.613, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.672, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.732, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.802, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.865, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.925, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 2.983, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.041, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.097, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.155, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.213, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.281, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.341, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.4, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.456, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.517, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.579, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.637, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.692, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.752, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.809, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.869, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.93, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 3.99, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.05, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.106, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.168, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.232, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.292, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.351, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.406, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.461, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.521, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.581, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.64, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.697, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.758, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.823, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.884, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 4.942, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.001, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.058, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.118, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.175, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.234, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.292, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.35, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.41, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.471, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.53, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.586, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.645, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.705, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.768, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.83, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.894, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 5.954, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.019, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.078, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.137, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.196, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.257, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.313, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.37, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.431, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.492, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.557, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.625, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.688, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.746, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.807, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.865, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.921, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 6.982, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.042, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.1, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.16, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.222, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.28, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.345, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.402, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.464, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.519, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.581, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.643, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.705, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.772, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.838, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.901, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 7.962, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.03, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.097, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.206, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.312, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.416, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.519, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.63, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.737, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.822, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.885, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.943, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 8.999, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.061, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.125, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.189, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.252, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.324, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.403, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.467, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.527, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.585, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.646, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.707, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.768, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.829, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.892, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 9.953, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.011, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.069, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.141, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.199, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.274, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 10.335, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 10.391, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.457, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 10.518, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 10.582, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.645, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 10.708, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 10.77, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 10.831, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 10.892, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 10.955, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.016, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 11.075, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 11.137, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.198, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 11.263, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 11.324, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.39, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 11.452, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 11.51, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.568, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 11.625, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 11.68, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.741, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 11.806, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 11.929, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 11.988, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 12.053, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 12.119, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.186, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 12.256, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 12.321, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.386, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 12.452, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 12.515, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.585, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 12.651, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 12.718, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.782, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 12.85, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 12.91, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 12.973, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 13.036, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 13.097, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "at": 13.16, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 13.222, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 13.283, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 13.344, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 13.411, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 13.489, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 13.552, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 13.612, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 14.74, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 14.812, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 14.873, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 14.93, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 14.995, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.052, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.114, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.175, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.234, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.301, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.369, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.44, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.508, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.57, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.627, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.689, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.749, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.813, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.869, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.927, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 15.985, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.048, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.119, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.187, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.266, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.337, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.405, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.476, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.547, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.618, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.686, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.752, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.818, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.881, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 16.943, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.003, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.062, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.124, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.197, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.266, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.342, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.409, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.48, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.563, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.625, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.684, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.744, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.803, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.858, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.917, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 17.976, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.036, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.093, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.153, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.21, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.272, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.431, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.494, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.552, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.615, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.678, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.737, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.801, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.868, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 18.929, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 19.028, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 19.119, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 19.211, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 19.308, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 19.419, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 19.511, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 19.615, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 19.678, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 19.737, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 19.813, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 19.915, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.018, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.116, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.213, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.312, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.411, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.474, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.53, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.592, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.652, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.716, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.777, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.838, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.898, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 20.958, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.013, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.071, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.137, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.2, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.263, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.331, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.398, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.475, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.535, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.591, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.65, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.712, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.779, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.841, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.903, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 21.963, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.03, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.088, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.145, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.2, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.26, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.319, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.381, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.441, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.498, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.558, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.62, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.68, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.749, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.812, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.875, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.931, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 22.992, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.053, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.117, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.183, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.242, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.304, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.361, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.424, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.486, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.547, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.604, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.675, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.734, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.792, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.852, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.913, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 23.971, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.03, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.088, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.147, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.202, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.259, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.319, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.384, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.446, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.502, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.561, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.623, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.683, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.744, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.806, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.87, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.925, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 24.981, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.041, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.097, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.16, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.222, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.281, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.343, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.406, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.467, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.526, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.583, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.644, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.704, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.767, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.828, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.893, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 25.956, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.021, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.081, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.147, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.205, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.265, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.325, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.382, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.445, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.506, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.569, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.636, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.704, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.771, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 26.857, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 26.926, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "at": 26.991, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 27.059, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 27.123, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "at": 27.187, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 27.254, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 27.315, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "at": 27.376, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 27.436, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 27.497, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "at": 27.559, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 27.622, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 27.692, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "at": 27.756, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 27.817, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 27.883, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "at": 27.958, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 28.029, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 28.09, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "at": 28.151, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 28.212, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 28.276, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "at": 28.341, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 28.412, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 28.48, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "at": 28.542, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 28.609, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 28.67, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "at": 28.735, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 28.809, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 28.879, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "at": 28.952, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 29.02, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "at": 29.091, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "at": 29.157, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 29.222, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 29.287, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 29.363, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 29.425, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 29.487, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 29.545, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 29.607, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 30.732, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 30.812, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 30.916, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.012, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.069, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.128, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.192, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.267, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.344, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.41, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.481, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.663, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.729, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.786, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.85, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.911, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 31.969, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.031, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.091, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.154, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.216, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.277, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.341, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.407, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.472, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.54, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.607, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.676, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.746, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.817, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.88, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 32.943, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.009, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.081, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.149, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.213, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.28, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.345, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.419, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.476, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.532, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.593, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.652, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.713, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.773, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.841, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.908, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 33.97, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.033, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.093, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.15, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.21, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.268, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.328, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.392, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.458, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.528, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.594, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.663, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.731, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.797, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.859, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.922, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 34.986, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.054, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.12, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.187, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.254, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.323, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.391, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.455, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.512, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.571, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.642, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.702, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.764, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.827, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.897, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 35.962, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 36.075, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 36.166, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 36.265, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 36.367, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 36.465, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 36.567, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 36.671, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 36.734, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 36.792, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 36.858, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 36.918, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 36.977, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.045, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.112, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.18, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.25, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.312, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.376, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.441, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.503, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.565, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.625, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.685, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.747, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.81, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.886, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 37.962, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.026, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.088, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.152, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.218, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.281, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.345, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.408, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.467, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.527, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.589, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.654, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.725, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.796, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.864, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.93, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 38.996, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.065, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.13, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.196, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.263, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.33, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.396, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.461, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.529, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.595, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.658, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.72, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.784, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.843, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.903, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 39.963, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.025, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.093, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.161, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.228, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.293, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.35, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.409, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.469, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.528, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.59, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.653, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.714, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.778, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.839, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.908, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 40.967, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.031, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.096, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.21, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.315, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.414, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.514, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.61, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.667, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.725, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.783, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.843, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.905, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 41.967, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.037, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.102, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.165, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.23, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.303, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.368, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.426, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.482, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.543, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.601, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.671, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.736, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.802, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.869, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.932, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 42.998, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.063, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.131, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.199, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.262, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.331, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.398, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.464, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.533, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.602, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.665, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.728, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.789, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.854, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.92, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 43.984, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.051, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.118, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.179, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.246, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.313, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.378, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.437, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.497, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.553, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.612, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.674, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.734, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.798, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.864, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 44.932, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "at": 45.002, + "instance": "082d4ffc44af424d8befa7efa58c2081" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "before": [ + { + "role": "api", + "container": "69839f9f1529", + "pid": 35301, + "started_at": "2026-11-15T16:15:20.333089426Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 402653184, + "instance": "adb3bf3bda284356bfe1328423b66363" + }, + { + "role": "api", + "container": "2f77c0676d92", + "pid": 36232, + "started_at": "2026-11-15T16:15:49.135928759Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 402653184, + "instance": "98564c46e81b4949ad9d2bb29df70ed6" + }, + { + "role": "cache", + "container": "e44e44c0b312", + "pid": 856, + "started_at": "2026-11-15T16:04:08.466599796Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "lb", + "container": "4b5bda8bb42d", + "pid": 9874, + "started_at": "2026-11-15T16:07:56.068812346Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8083" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "f6c1860d783c", + "pid": 918, + "started_at": "2026-11-15T16:04:08.652994337Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + } + ], + "after": [ + { + "role": "api", + "container": "974927964c29", + "pid": 36739, + "started_at": "2026-11-15T16:16:01.468749125Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 402653184, + "instance": "082d4ffc44af424d8befa7efa58c2081" + }, + { + "role": "api", + "container": "ba6e293a4c15", + "pid": 37275, + "started_at": "2026-11-15T16:16:17.369283258Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 402653184, + "instance": "21fd709de1454ee5a0b997715ee1cb99" + }, + { + "role": "cache", + "container": "e44e44c0b312", + "pid": 856, + "started_at": "2026-11-15T16:04:08.466599796Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "lb", + "container": "4b5bda8bb42d", + "pid": 9874, + "started_at": "2026-11-15T16:07:56.068812346Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8083" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "f6c1860d783c", + "pid": 918, + "started_at": "2026-11-15T16:04:08.652994337Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + } + ] + }, + "replay_after_replacement": true, + "public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 115, \"failed_attempts\": 3, \"first_recovery_seconds\": 3.142}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.", + "after_persistence": [ + { + "role": "api", + "container": "974927964c29", + "pid": 40556, + "started_at": "2026-11-15T16:17:33.698835293Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 402653184, + "instance": "a5f6c6aecd8d4701978b03a282d5388b" + }, + { + "role": "api", + "container": "ba6e293a4c15", + "pid": 40592, + "started_at": "2026-11-15T16:17:33.728153293Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 402653184, + "instance": "d08fb8a544ab4ba9b8563011e07b2185" + }, + { + "role": "cache", + "container": "e44e44c0b312", + "pid": 40364, + "started_at": "2026-11-15T16:17:33.055158543Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "lb", + "container": "4b5bda8bb42d", + "pid": 40318, + "started_at": "2026-11-15T16:17:33.013708751Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8083" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "f6c1860d783c", + "pid": 40338, + "started_at": "2026-11-15T16:17:33.031002126Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + } + ] +} diff --git a/evidence/http-tests.json b/evidence/http-tests.json new file mode 100644 index 00000000..94ab8ac9 --- /dev/null +++ b/evidence/http-tests.json @@ -0,0 +1,15 @@ +{ + "date": "2026-11-15T16:16:15.978617+00:00", + "source_commit": "b14fb9a162d8012e83699cd8facdab178587d04b", + "runtime_image_revision": "df4eee9", + "command": "BASE_URL=http://127.0.0.1:8083 .venv/bin/python -m pytest student-tests -q --tb=short", + "result": "PASS", + "output": "........................................................................ [ 82%]\n............... [100%]\n87 passed in 279.66s (0:04:39)", + "prior_attempt": { + "result": "INVALID_ENVIRONMENT", + "passed": 27, + "failed": 19, + "errors": 41, + "reason": "Test requests overlapped with a mistaken full Compose image recreation; LB/API were unavailable. Repeated after completion without concurrent mutations." + } +} diff --git a/research/run.py b/research/run.py index 99c446c6..8a672bc4 100644 --- a/research/run.py +++ b/research/run.py @@ -95,6 +95,21 @@ async def load(base, token, chat, concurrency, duration): } +def cpu_counters(env): + ids = compose(env, "ps", "-q", "api").split() + code = ( + "import json,pathlib; print(json.dumps({p: " + "pathlib.Path('/sys/fs/cgroup/'+p).read_text() for p in " + "['cpu.stat','memory.current','memory.peak']}))" + ) + return { + cid[:12]: json.loads( + command(["docker", "exec", cid, "python", "-c", code], env) + ) + for cid in ids + } + + def sample(env, stop, output): while not stop.is_set(): try: @@ -277,6 +292,7 @@ def main(): ) assert r.status_code == 201 asyncio.run(load(base, token, chat, c, a.warmup)) + counters_before = cpu_counters(env) samples = [] stop = threading.Event() thread = threading.Thread(target=sample, args=(env, stop, samples)) @@ -286,6 +302,7 @@ def main(): finally: stop.set() thread.join() + counters_after = cpu_counters(env) ids = compose(env, "ps", "-q").split() inspect = json.loads(command(["docker", "inspect", *ids], env)) result.update( @@ -298,6 +315,8 @@ def main(): api_memory_mib_each=memory, dataset_initial_messages=100, samples=samples, + cgroup_before=counters_before, + cgroup_after=counters_after, actual_limits=[ { "role": x["Config"]["Labels"][