From 5c8f605cd3bd40245109348463d3e7111379b2d3 Mon Sep 17 00:00:00 2001 From: Artyom Date: Thu, 5 Nov 2026 22:33:29 +0300 Subject: [PATCH] Archive grade 4 verification and record SQLAlchemy failure evidence --- REPORT.md | 145 +- evidence/README.md | 66 +- evidence/detection/run1/kill.json | 12 +- evidence/detection/run1/pause.json | 14 +- evidence/detection/run1/postgres.json | 12 +- evidence/detection/run1/sigterm.json | 12 +- evidence/detection/run2/kill.json | 12 +- evidence/detection/run2/pause.json | 12 +- evidence/detection/run2/postgres.json | 12 +- evidence/detection/run2/sigterm.json | 12 +- evidence/detection/run3/kill.json | 12 +- evidence/detection/run3/pause.json | 14 +- evidence/detection/run3/postgres.json | 14 +- evidence/detection/run3/sigterm.json | 12 +- .../failover-each/5bbf259ff136/action.txt | 15 + evidence/failover-each/5bbf259ff136/k6/k6.txt | 3 + .../5bbf259ff136/k6/summary.json | 309 +++ evidence/failover-each/5bbf259ff136/lb_up.csv | 657 ++++++ evidence/failover-each/5bbf259ff136/probe.csv | 644 ++++++ .../result.json | 39 +- .../failover-each/7cfd91ddf780/action.txt | 15 + evidence/failover-each/7cfd91ddf780/k6/k6.txt | 3 + .../7cfd91ddf780/k6/summary.json | 309 +++ evidence/failover-each/7cfd91ddf780/lb_up.csv | 663 ++++++ evidence/failover-each/7cfd91ddf780/probe.csv | 647 ++++++ .../result.json | 39 +- .../failover-each/fe75c6c37e5a/action.txt | 14 - evidence/failover-each/fe75c6c37e5a/k6/k6.txt | 2 - .../fe75c6c37e5a/k6/summary.json | 309 --- evidence/failover-each/fe75c6c37e5a/lb_up.csv | 653 ------ evidence/failover-each/fe75c6c37e5a/probe.csv | 647 ------ .../failover-each/ff81f93e8b0d/action.txt | 14 - evidence/failover-each/ff81f93e8b0d/k6/k6.txt | 2 - .../ff81f93e8b0d/k6/summary.json | 309 --- evidence/failover-each/ff81f93e8b0d/lb_up.csv | 655 ------ evidence/failover-each/ff81f93e8b0d/probe.csv | 647 ------ evidence/failover-each/summary.json | 254 +-- evidence/grade4-baseline-proxy-failed.txt | 8 + evidence/grade4-baseline.txt | 66 + evidence/grade4-checks.txt | 35 + .../grade4-detection-proxy-interrupted.txt | 65 + .../run1/sigterm.json | 35 + evidence/grade4-detection.txt | 478 +++++ evidence/grade4-development.txt | 21 + evidence/grade4-drain.txt | 25 + .../grade4-evidence-audit-fixture-token.txt | 48 + .../grade4-evidence-audit-scan-failed.txt | 53 + evidence/grade4-evidence-audit.txt | 43 + evidence/grade4-failover-each.txt | 26 + evidence/grade4-final-review.txt | 33 + evidence/grade4-limits-compose-rejection.txt | 6 + .../grade4-limits-negative-config-failed.txt | 20 + evidence/grade4-limits-negative.txt | 45 + evidence/grade4-migration-repeat.txt | 26 + evidence/grade4-persistence-scoped.txt | 158 ++ evidence/grade4-public-events-unavailable.txt | 28 + evidence/grade4-public.txt | 30 + evidence/grade4-rolling-from-7fc911c.txt | 26 + evidence/grade4-rolling.txt | 10 + evidence/grade4-runtime-limits.txt | 19 + evidence/grade4-start.txt | 146 ++ evidence/grade4-stop-first.txt | 37 + evidence/grade4-stop.txt | 37 + evidence/grade4-student-tests.txt | 303 +++ evidence/lb-check.json | 10 +- evidence/limits-violations.txt | 42 +- evidence/limits.txt | 12 +- evidence/rolling-from-7fc911c/action.txt | 35 + evidence/rolling-from-7fc911c/k6/k6.txt | 3 + evidence/rolling-from-7fc911c/k6/summary.json | 309 +++ evidence/rolling-from-7fc911c/lb_up.csv | 936 +++++++++ evidence/rolling-from-7fc911c/probe.csv | 915 ++++++++ evidence/rolling-from-7fc911c/result.json | 66 + evidence/rolling/action.txt | 70 +- evidence/rolling/k6/k6.txt | 3 +- evidence/rolling/k6/summary.json | 610 +++--- evidence/rolling/lb_up.csv | 1842 +++++++++-------- evidence/rolling/probe.csv | 1833 ++++++++-------- evidence/rolling/result.json | 31 +- .../design.md | 6 +- .../tasks.md | 19 +- .../.openspec.yaml | 4 + .../design.md | 0 .../proposal.md | 0 .../specs/lab3-failover-operations/spec.md | 0 .../tasks.md | 16 +- .../specs/lab3-failover-operations/spec.md | 26 +- 87 files changed, 10063 insertions(+), 5762 deletions(-) create mode 100644 evidence/failover-each/5bbf259ff136/action.txt create mode 100644 evidence/failover-each/5bbf259ff136/k6/k6.txt create mode 100644 evidence/failover-each/5bbf259ff136/k6/summary.json create mode 100644 evidence/failover-each/5bbf259ff136/lb_up.csv create mode 100644 evidence/failover-each/5bbf259ff136/probe.csv rename evidence/failover-each/{fe75c6c37e5a => 5bbf259ff136}/result.json (59%) create mode 100644 evidence/failover-each/7cfd91ddf780/action.txt create mode 100644 evidence/failover-each/7cfd91ddf780/k6/k6.txt create mode 100644 evidence/failover-each/7cfd91ddf780/k6/summary.json create mode 100644 evidence/failover-each/7cfd91ddf780/lb_up.csv create mode 100644 evidence/failover-each/7cfd91ddf780/probe.csv rename evidence/failover-each/{ff81f93e8b0d => 7cfd91ddf780}/result.json (59%) delete mode 100644 evidence/failover-each/fe75c6c37e5a/action.txt delete mode 100644 evidence/failover-each/fe75c6c37e5a/k6/k6.txt delete mode 100644 evidence/failover-each/fe75c6c37e5a/k6/summary.json delete mode 100644 evidence/failover-each/fe75c6c37e5a/lb_up.csv delete mode 100644 evidence/failover-each/fe75c6c37e5a/probe.csv delete mode 100644 evidence/failover-each/ff81f93e8b0d/action.txt delete mode 100644 evidence/failover-each/ff81f93e8b0d/k6/k6.txt delete mode 100644 evidence/failover-each/ff81f93e8b0d/k6/summary.json delete mode 100644 evidence/failover-each/ff81f93e8b0d/lb_up.csv delete mode 100644 evidence/failover-each/ff81f93e8b0d/probe.csv create mode 100644 evidence/grade4-baseline-proxy-failed.txt create mode 100644 evidence/grade4-baseline.txt create mode 100644 evidence/grade4-checks.txt create mode 100644 evidence/grade4-detection-proxy-interrupted.txt create mode 100644 evidence/grade4-detection-proxy-interrupted/run1/sigterm.json create mode 100644 evidence/grade4-detection.txt create mode 100644 evidence/grade4-development.txt create mode 100644 evidence/grade4-drain.txt create mode 100644 evidence/grade4-evidence-audit-fixture-token.txt create mode 100644 evidence/grade4-evidence-audit-scan-failed.txt create mode 100644 evidence/grade4-evidence-audit.txt create mode 100644 evidence/grade4-failover-each.txt create mode 100644 evidence/grade4-final-review.txt create mode 100644 evidence/grade4-limits-compose-rejection.txt create mode 100644 evidence/grade4-limits-negative-config-failed.txt create mode 100644 evidence/grade4-limits-negative.txt create mode 100644 evidence/grade4-migration-repeat.txt create mode 100644 evidence/grade4-persistence-scoped.txt create mode 100644 evidence/grade4-public-events-unavailable.txt create mode 100644 evidence/grade4-public.txt create mode 100644 evidence/grade4-rolling-from-7fc911c.txt create mode 100644 evidence/grade4-rolling.txt create mode 100644 evidence/grade4-runtime-limits.txt create mode 100644 evidence/grade4-start.txt create mode 100644 evidence/grade4-stop-first.txt create mode 100644 evidence/grade4-stop.txt create mode 100644 evidence/grade4-student-tests.txt create mode 100644 evidence/rolling-from-7fc911c/action.txt create mode 100644 evidence/rolling-from-7fc911c/k6/k6.txt create mode 100644 evidence/rolling-from-7fc911c/k6/summary.json create mode 100644 evidence/rolling-from-7fc911c/lb_up.csv create mode 100644 evidence/rolling-from-7fc911c/probe.csv create mode 100644 evidence/rolling-from-7fc911c/result.json rename openspec/changes/{verify-lab3-grade4-recovery => archive/2026-11-05-verify-lab3-grade4-recovery}/.openspec.yaml (57%) rename openspec/changes/{verify-lab3-grade4-recovery => archive/2026-11-05-verify-lab3-grade4-recovery}/design.md (100%) rename openspec/changes/{verify-lab3-grade4-recovery => archive/2026-11-05-verify-lab3-grade4-recovery}/proposal.md (100%) rename openspec/changes/{verify-lab3-grade4-recovery => archive/2026-11-05-verify-lab3-grade4-recovery}/specs/lab3-failover-operations/spec.md (100%) rename openspec/changes/{verify-lab3-grade4-recovery => archive/2026-11-05-verify-lab3-grade4-recovery}/tasks.md (66%) diff --git a/REPORT.md b/REPORT.md index 5afdf195..437ca4e8 100644 --- a/REPORT.md +++ b/REPORT.md @@ -2,13 +2,13 @@ - Автор, группа: Костылев Артём, БПИ2303 - Целевая оценка: **5**; трек «5» — **трек 1 «Цена масштабирования»** -- Ветка: `solution/bpi2303-kostylev`. Текущий этап — повторная проверка уровня «3» после перехода - на SQLAlchemy 2.0.45 и Alembic 1.16.5; код прогонов — `0a1f48bd73e5780c732356880c23fdb476bcff98`. - **Уровень «3» подтверждён** на этом коде; после него меняются только отчёт, evidence и OpenSpec. +- Ветка: `solution/bpi2303-kostylev`. Уровень «3» подтверждён на + `0a1f48bd73e5780c732356880c23fdb476bcff98`; этап grade4 повторяет проверки уровня «4» на + SQLAlchemy 2.0.45 / Alembic 1.16.5, код `252e3e3ffeea5004cfd9dc342e686af1417a77a2`. Публикация в этом этапе не выполняется. -- Уровень «4» и трек ниже пока содержат **исторические результаты raw asyncpg**. Они не подтверждают - текущую производительность и отказоустойчивость под нагрузкой. Активный change - `2026-11-05-adopt-sqlalchemy-alembic` остаётся открытым до последовательных этапов «4» и трека. +- Трек «Цена масштабирования» ниже пока содержит **исторические результаты raw asyncpg**, + которые не подтверждают производительность SQLAlchemy. Активный change + `2026-11-05-adopt-sqlalchemy-alembic` остаётся открытым до этапа трека. - Источники и инструменты: FastAPI, gunicorn + uvicorn-worker, dishka, pydantic, structlog, SQLAlchemy, Alembic, asyncpg как драйвер, argon2-cffi, redis-py, pytest, ruff, ty; версии — `evidence/environment.txt` и `uv.lock`. Документация установленного SQLAlchemy проверена через @@ -25,8 +25,9 @@ ```sh git clone && cd mtusi2026lab3 && git checkout solution/bpi2303-kostylev -export COMPOSE_PROJECT_NAME=mtusi2026lab3-grade3 +export COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4 export LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +export NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 make secrets # secrets/*: пароли PostgreSQL и Valkey, DATABASE_URL, CACHE_URL, ключ курсора; .env из .env.example docker compose --env-file .env config --quiet @@ -59,7 +60,7 @@ make down # безопасная остановка: контейне прогон сразу после первого попадает в ту же реплику, поэтому перед ним нужен один запрос через LB (строка `curl …/health/live` выше или `make failover-evidence SHIFT=1`). -Прогон текущего этапа выполнен из рабочего checkout, на новом Compose-проекте +Прогон уровня «3» выполнен из рабочего checkout, на новом Compose-проекте `mtusi2026lab3-grade3-final-20261105` и пустых volumes, с указанными портами и без явного `BASE_URL`. `evidence/grade3-start.txt` содержит пустой список volumes перед запуском, сборку, `make up`, `docker compose ps -a` и версию Alembic `0002_idempotency_keys`. Старые volumes не удалялись. @@ -91,7 +92,7 @@ make limits # лимиты, порты, секреты, volumes, make lb-check # 40 GET одним токеном + 20 раундов × 8 параллельных POST с одним ключом make detection # три прогона: SIGTERM, kill, pause, остановка PostgreSQL → evidence/detection/run{1,2,3}/ make failover-each # каждая реплика: остановка, 10 с без неё, возврат — под k6 100 rps -make rolling-update FROM_REV=8d28d45 # стенд на версии уровня «3», затем замена на текущую под нагрузкой +make rolling-update FROM_REV=7fc911c # предыдущая SQLAlchemy-версия → текущая под нагрузкой make rolling-update # замена текущей сборкой (эквивалентная поочерёдная замена) ``` @@ -214,11 +215,9 @@ SHA-256 канонического JSON распарсенной модели, завершённого к этому сроку, отменяет и выходит с кодом 0. Отмену ловит конвейер: пишет `request.cancelled` и, если ответ ещё не начат, отвечает `503 unavailable` в общем формате с `X-Request-Id` и `X-Instance-Id`, затем пробрасывает отмену дальше (без этого uvicorn отвечал своим `text/plain 500` без заголовков контракта); весь срок — ≤5 с < gunicorn `graceful_timeout` 6 с < -`docker stop -t 7` хука < `stop_grace_period: 8s` < 10 с, поэтому мастер gunicorn не доходит до `SIGKILL` воркера. Без окна (уровень «3») gunicorn закрывал listener сразу, и запрос, уже -стоявший в очереди `accept`, LB видел как обрыв после отправки: для `GET` и сообщений с ключом это повтор, для -`POST /users`, `/chats` и входа — `502`. Разница видна в счётчиках LB: rolling update со старой версии — -`wredis` 56 (запросы, перенаправленные на соседа), с новой — все 0 (`evidence/rolling*/result.json`). Хук -`stop-one.sh` теперь идёт 2,66 с (`evidence/hooks.txt`). На реплику `cpus: 1`, `512m`, 2 воркера, пул 8: 2 × 2 × 8 = 32 соединения < +`docker stop -t 7` хука < `stop_grace_period: 8s` < 10 с, поэтому мастер gunicorn не доходит до `SIGKILL` воркера. Результаты текущего кода и счётчики повторов LB приведены в таблице уровня «4» ниже. +Хук останавливает конкретный контейнер, соответствие hostname проверяется через `docker inspect`. +На реплику `cpus: 1`, `512m`, 2 воркера, пул 8: 2 × 2 × 8 = 32 соединения < `max_connections=200`. API работает `read_only` с `tmpfs /tmp`. **Проверки, период и максимум обнаружения.** `GET /health/ready` раз в `inter 1s`; после первого провала — @@ -229,21 +228,21 @@ SHA-256 канонического JSON распарсенной модели, gunicorn — первый прогон `failover-each` показал `wredis` 19 и 49 без ошибок; это записано в `design.md` изменения `2026-10-28-add-lab3-failover-operations`, сырьё того прогона заменено повторным с `init-state`). -Следующая таблица содержит исторические замеры до SQLAlchemy; текущий этап не повторяет `make detection`. +Три серии на SQLAlchemy 2.0.45 (`252e3e3`), команда `make detection`: -| Отказ | Как замечает LB | Максимум обнаружения | Замер (`make detection`) | +| Отказ | Как замечает LB | Максимум обнаружения | Замер, с | | --- | --- | --- | --- | -| `SIGTERM` (`docker stop`, `stop-one.sh`) | readiness `503` в окне вывода | `inter + (fall−1)·fastinter` = 1,5 с, и раньше закрытия порта (2 с) | 1,26–1,35 с; действие (`docker stop -t 7`) завершилось через 2,47–2,56 с | -| процесс убит (`docker kill`) | отказ соединения, DNS | 1,5 с | 0,42–0,53 с | -| процесс завис (`docker pause`) | таймаут проверки | `inter + (fall−1)·fastinter + fall·timeout check` = 3,5 с | 2,93–3,06 с | -| PostgreSQL остановлен | readiness всех реплик `503` | 1,5 с до 0 `UP`; LB сам отвечает `503` | 1,26–1,27 с; `/health/live` через LB → `503 unavailable` без `X-Instance-Id` | +| `SIGTERM` (`docker stop -t 7`) | readiness `503` в окне вывода | 1,5 с | 1.069–1.266; остановка 2.535–2.665 | +| `docker kill` | отказ соединения, DNS | 1,5 с | 0.886–0.934 | +| `docker pause` | таймаут проверки | 3,5 с | 2.778–3.215 | +| Остановка PostgreSQL | readiness всех реплик `503` | 1,5 с до 0 `UP` | 1.196–1.263; через LB `503 unavailable` без `X-Instance-Id` | -Замер — три прогона `make detection` подряд (`evidence/detection/run{1,2,3}/`), в таблице диапазон. Время -считается от запуска команды отказа, поэтому замер включает задержку docker CLI до сигнала и период опроса -статистики LB (50 мс) и может превысить формулу на эту величину: допуск замера — 0,25 с. На коде до SQLAlchemy все 12 -замеров уложились в формулу без допуска (в историческом `faults.txt` остановка PostgreSQL — 1,04 с); раньше допуск -понадобился: на `a02d442` у PostgreSQL в двух прогонах из трёх (1,553 и 1,551 с), при ревью — у `SIGTERM` (1,609 с), -на `e00eb6c` — у `SIGTERM` в `failover-each` (1,515 с). Возврат в ротацию после восстановления — 2,35–3,56 с. +Для ответившей `503` реплики максимум — `inter + (fall−1)·fastinter` = 1,5 с; +для зависшей — дополнительно `fall·timeout check`, итого 3,5 с. Измерение начинается перед +запуском docker CLI, статистика LB опрашивается раз в 50 мс; допуск — 0,25 с. Все 12 замеров +уложились в формулу даже без допуска, SIGTERM исключён до завершения остановки. +Возврат после команды восстановления — 1.953–3.095 с. +Данные: `evidence/detection/run{1,2,3}/*.json`, команда и SHA — `evidence/grade4-detection.txt`. **Цепочка таймаутов** (проверяется тестом `student-tests/unit/test_timeouts.py` по `compose.yaml`, `haproxy.cfg` и `scripts/stop-one.sh`, а не только таблицей): @@ -308,16 +307,16 @@ postgres 2/1g, valkey 0.5/128m, k6 2/512m (`make limits` сверяет их и ## Доказательства критериев -Все текущие проверки выполнены 2026-11-05 на `0a1f48bd73e5780c732356880c23fdb476bcff98`. +Проверки уровня «3» выполнены 2026-11-05 на `0a1f48bd73e5780c732356880c23fdb476bcff98`. Стенд — `mtusi2026lab3-grade3-final-20261105`, LB `127.0.0.1:18083`, статистика `23083`; `BASE_URL` автоматически определяется Makefile. Даты, команды, коды завершения и SHA — в `evidence/`. | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | | **3.1** Документированный запуск, ≥2 API, общая БД и сессии, одна точка входа | `make up`; `make stack-evidence` | Новые пустые volumes; `migrate` завершился с 0, версия `0002_idempotency_keys`; обе API healthy/UP, один PostgreSQL и Valkey; API без опубликованных портов, только LB на 18083/23083; `container_name` отсутствует | `evidence/grade3-start.txt`, `evidence/stack.txt`, `evidence/environment.txt` | -| **3.2** Один токен работает через разные реплики | `make lb-check` | 40 GET: `1b4a73f61b07` и `8ffe0c70f4ea` по 20 ответов; hostname совпадают с двумя контейнерами | `evidence/lb-check.json`, `evidence/grade3-lb-check.txt`, `evidence/stack.txt` | +| **3.2** Один токен работает через разные реплики | `make lb-check` | 40 GET: `1b4a73f61b07` и `8ffe0c70f4ea` по 20 ответов; hostname совпадают с двумя контейнерами | `evidence/grade3-lb-check.txt` (актуальный `lb-check.json` повторно снят на этапе grade4), `evidence/stack.txt` | | **3.3** Остановка любой одной реплики, чтение/запись ≤10 с | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`; `docker compose ps -a`; `make restore`; сдвиг round robin и повтор | По очереди остановлены `api-1` (`8ffe0c70f4ea`) и `api-2` (`1b4a73f61b07`), обе с `Exited (0)`; сосед продолжал работать. Первая запись после хука через 2.762 / 2.817 с, 129 / 128 успешных записей, 0 / 0 ошибок, по 2 наблюдаемых реплики; история подтверждённых сообщений сохранена | `evidence/failover-current-1.txt`, `evidence/failover-current-2.txt`, `evidence/grade3-restore-1.txt`, `evidence/grade3-restore-2.txt` | -| **3.4** Конкурентный одинаковый ключ и смена реплики | `make lb-check` | 20 раундов × 8 POST: 160 ответов 201, по 80 от каждой реплики; 20/20 одинаковых тел, одна копия в истории, другой body → 409 | `evidence/lb-check.json`, `evidence/grade3-lb-check.txt` | +| **3.4** Конкурентный одинаковый ключ и смена реплики | `make lb-check` | 20 раундов × 8 POST: 160 ответов 201, по 80 от каждой реплики; 20/20 одинаковых тел, одна копия в истории, другой body → 409 | `evidence/grade3-lb-check.txt` (актуальный `lb-check.json` повторно снят на этапе grade4) | | **3.4** Безопасный повтор неподтверждённой записи | `make retry-check` | При блокировке чата 1.5 с клиент прекращает ожидание через 0.5 с. Повтор с тем же ключом на другой реплике: 201 с первой попытки, одна копия и тот же ID. Оба варианта — живая реплика и `docker kill` ожидающей реплики — прошли; повтор через 1.539 / 1.551 с | `evidence/retry.json`, `evidence/grade3-retry.txt` | | **3.5** Публичный smoke, persistence и failover | `make test`; `make persistence-evidence` (вызывает `make test-persistence ACTION_SCRIPT=scripts/restart.sh`); два failover выше | Все PASS. При persistence прежний токен и история сохранились, IDs контейнеров те же, StartedAt у lb, обеих API, postgres, valkey и migrate обновились | `evidence/smoke.txt`, `evidence/persistence.txt`, `evidence/grade3-persistence.txt`, `evidence/failover-current-*.txt` | | Перенос lab2: миграции, схема и повтор без изменения данных | `make up` на пустых volumes; `make storage-evidence` | PK/FK/UNIQUE/CHECK и индексы показаны в схеме; Alembic head `0002_idempotency_keys`. Повтор миграций: код 0, до/после 70 users, 24 sessions, 6 chats, 12 members, 284 messages и 284 keys, 1 строка версии | `evidence/schema.txt`, `evidence/migrations.txt`, `evidence/grade3-storage.txt` | @@ -327,28 +326,60 @@ postgres 2/1g, valkey 0.5/128m, k6 2/512m (`make limits` сверяет их и | Контракт, OpenAPI, форматирование и типы | `make check`; `make validate`; `make lint`; `uv lock --check`; `openspec validate --all --strict` | Все успешно; 107 файлов соответствуют формату. Diff нормативных файлов против upstream пуст | `evidence/checks.txt` | | Остановка без удаления данных | `make down`; `docker compose ps -a`; `docker volume ls --filter label=…` | Контейнеров этапа нет; оба volumes финального и исходного прогонов сохранены | `evidence/grade3-stop.txt`, `evidence/grade3-initial-stop.txt` | -**Уровень «4» — исторические прогоны `e049c26`, до SQLAlchemy.** Повторная проверка -текущего кода выполняется следующим этапом. Приведённые ниже результаты не заявляются как актуальные. +**Уровень «4» подтверждён на SQLAlchemy**, код `252e3e3ffeea5004cfd9dc342e686af1417a77a2`. +Проект `mtusi2026lab3-grade4-20261105`, порты `18083/23083`; запуск и версии — `grade4-start.txt`. +Старые замеры raw asyncpg доступны через `git show ef0b9a3:evidence/<имя>`. -Исторические версии перезаписанных файлов доступны как `git show 7fc911c:evidence/<имя>`; -текущие файлы уровня «3» содержат новые прогоны. +Нагрузка — k6 open model из `edge` в `lb:8080`: 100 запросов/с, 80 % отправок сообщений, +20 % чтений истории, 20 пользователей и 10 чатов, прогрев 10 с, замер 60/90 с. +Отдельный зонд пишет каждые 0,1 с; после ошибки повторяет тот же ключ, после прогона читает +все страницы своего чата и сверяет тела Message. Время восстановления теперь считается +по завершению успешного ответа после последней ошибки; отсутствие такого ответа, дубли, +потери, dropped iterations или отсутствующая статистика LB делают прогон неуспешным. +14 регрессионных тестов покрывают эти случаи. В итоговых прогонах ошибок зонда нет, +поэтому recovery_s = 0; время первого успешного ответа после начала действия указано отдельно. -Нагрузка — k6 open model из сети `edge` в `lb:8080`: 100 запросов/с постоянной -интенсивности (80 % `POST` сообщения с уникальным ключом, 20 % чтения первой страницы истории), 20 -пользователей и 10 чатов, прогрев 10 с, замер 60–90 с; параллельно зонд шлёт сообщение каждые 0,1 с и после -ошибки повторяет тот же ключ, а опрос статистики LB 10 раз в секунду пишет число `UP`. Счётчики LB — -дельта за прогон по backend `api`: `wretr` (повторы), `wredis` (перенаправления), `econ`, `eresp`, `hrsp_5xx`. - -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий README | Команда | Наблюдение | Доказательство | | --- | --- | --- | --- | -| **4.1** Health/readiness, таймауты и прекращение приёма при `SIGTERM` согласованы | `make student-test` (`test_timeouts.py`, `test_drain.py`, вывод `-s`) | цепочка: LB server 10 с > запрос 8 с > БД 3 с; readiness 0,4 с < check 1 с; окно 2 с > обнаружение 1,5 с; дообработка 3 с ≥ БД 3 с; остановка 6 с < хук 7 с < `stop_grace_period` 8 с; настоящий gunicorn по `SIGTERM` (в тесте окно 1 с, срок 3 с): ready/live `(503, 200)` в окне, порт закрыт через 1.17 с, запрос, принятый до сигнала и дописанный после закрытия порта, получил `201`, запрос, тело которого так и не пришло, прерван ответом `503 unavailable` в общем формате (`application/json`, `request_id` = `X-Request-Id`, есть `X-Instance-Id`), выход через 2.43 с | `evidence/student-tests.txt`, `compose.yaml`, `infra/haproxy/haproxy.cfg`, `src/messenger/gunicorn_conf.py` | -| **4.1** LB исключает недоступный backend; период проверок и максимум обнаружения | `make detection` (три прогона) | период 1 с (0,5 с после провала), `fall 2`/`rise 2`; заявлено ≤1,5 с (`503`/отказ соединения) и ≤3,5 с (зависание), допуск замера 0,25 с; измерено (min–max трёх прогонов): `SIGTERM` 1.259–1.348 с — раньше конца остановки (2.471–2.555 с), `kill` 0.420–0.526 с, `pause` 2.926–3.063 с, PostgreSQL 1.263–1.266 с до 0 `UP` и `503 unavailable` от LB; возврат 2.347–3.555 с; все 12 `pass: true`, все в пределах формулы без допуска | `evidence/detection/run{1,2,3}/{sigterm,kill,pause,postgres}.json` | -| **4.2** Последовательно остановить каждую реплику (с восстановлением предыдущей); RPS, ошибки, время восстановления | `make failover-each` (на реплику: `stop-one.sh`, `docker compose ps -a` реплик, 10 с без неё, `restore.sh` — всё внутри окна нагрузки) | `ff81f93e8b0d`: 100.02 rps, 6001 запрос k6, ошибок 0, p50/p95/p99 1.60/3.92/7.31 мс, обнаружение 1.405 с, возврат в ротацию через 14.834 с от начала действия (действие 15.296 с: хук + 10 с + restore), зонд 646 попыток / 0 ошибок, пока реплика исключена, её `X-Instance-Id` не встречался (121 ответ только от `fe75c6c37e5a`); `fe75c6c37e5a`: 100.02 rps, 6001, 0 ошибок, 1.57/3.62/6.24 мс, 0.859 с, 14.343 с, 646/0; в `action.txt` сразу после хука — `Exited (0)` у остановленной и `Up (healthy)` у второй, затем снимок без остановленной реплики: база 9,7 и 12,9 МБ (2 759 и 9 053 сообщения по `n_live_tup`), оставшаяся реплика 12,8 и 22,8 % CPU, 126 и 142 МиБ из 512, PostgreSQL ≈4 % и 49–55 МиБ; счётчики LB `wretr`/`wredis`/`econ`/`eresp`/`hrsp_5xx` — 0. Обнаружение ≤1,5 с без допуска (на `e00eb6c` было 1.515 с — в пределах допуска замера 0,25 с на docker CLI и опрос, см. `make detection`). Один прогон на реплику: критерий бинарный (0 ошибок), а остановки повторены в failover-тестах (`failover-*.txt`, `clean-clone.txt`) и `make test-faults` — неудачных записей нет ни в одном | `evidence/failover-each/summary.json`, `evidence/failover-each//` | -| **4.2** Общая идемпотентность — параллельными запросами с одним ключом | `make lb-check` | 20 раундов × 8: 160 × `201` от двух реплик, 20/20 раундов с одинаковыми телами, одной копией и `409` на другой body; на коде до SQLAlchemy — то же | `evidence/lb-check.json`, `evidence/clean-clone.txt`, `evidence/track1/after-checks.txt` | -| **4.3** CPU/RAM-лимиты, API без портов в обход LB, секреты и хранилища только где нужно, нет `container_name` | `make limits` (`docker compose --profile bench config` + `docker inspect` против политики в `scripts/limits.py`) на коде до SQLAlchemy | лимиты у всех 6 сервисов (api 1/512MiB, lb 1/128MiB, postgres 2/1024MiB, valkey 0.5/128MiB, migrate 0.5/256MiB, k6 2/512MiB) и применены у запущенных; порты только у `lb` на `127.0.0.1`; секреты: api — 3, migrate — `database_url`, postgres — пароль, valkey — конфиг, lb и k6 — нет; volumes `pgdata`/`valkeydata` только у своих хранилищ; `backend` internal, k6 только в `edge`; api и lb `read_only`; `PASS`. Подмешанные нарушения (порт и лишний секрет у api, `container_name`, снятый лимит) → 5 нарушений, код 1 | `evidence/limits.txt`, `evidence/limits-violations.txt` | -| **4.4** Rolling update под нагрузкой | `make rolling-update FROM_REV=8d28d45` (стенд на образе уровня «3» с тегом `mtusi-messenger:8d28d45`, затем замена на текущую версию с запасом +1) | `lb_up_min` 2 из 2, k6 100 rps, 8996 запросов, ошибок 0 (5 итераций k6 сброшены), зонд 913/0; после замены отвечают только новые контейнеры (`3e741644dba4`, `3e62ff0322c6` → `6c2a02267340`, `158dfc1bd95d`; в `action.txt` у каждой замены — тег и ID образа: `mtusi-messenger:8d28d45@75eeba4b5bfd` → `mtusi-messenger:mtusi2026lab3@c69a721bc974`); база 16 МБ / 15 139 сообщений, реплики по 20 % CPU и 145 МиБ; старые реплики без окна дали `wredis` 56 — запросы, перенаправленные LB на соседа | `evidence/rolling-from-8d28d45/result.json`, `action.txt` | -| **4.4** То же на текущей версии | `make rolling-update` | `lb_up_min` 2, 9001 запрос, ошибок 0, зонд 919/0, все реплики новые (`6c2a02267340`, `158dfc1bd95d` → `c2f5d82dee7c`, `65d10d346f4e`); все счётчики LB 0. Образ пересобран из той же ревизии, тег тот же, ID другой (`mtusi-messenger:mtusi2026lab3@c69a721bc974` → `@021faac1e94a`): BuildKit пишет при сборке новый манифест с аттестацией (`exporting attestation manifest` в выводе сборки `clean-clone.txt`) — это эквивалентная поочерёдная замена | `evidence/rolling/result.json`, `action.txt` | -| **4.4** Миграции однократно и совместимы с работающей версией | `scripts/rolling-update.sh`; `make storage-evidence` | `migrate` запускается один раз до замены первой реплики (`migration.completed up to date`), реплики схему не трогают, `migrate` под `pg_advisory_lock`; между уровнями «3» и «4» схема не менялась; `0002_idempotency_keys` только добавляет таблицу, код лабы 2 её не замечает | `evidence/rolling*/action.txt`, `evidence/migrations.txt` | +| **4.1** Readiness, SIGTERM и таймауты | `make student-test PYTEST_ARGS='-v -s student-tests/api/test_drain.py student-tests/unit/test_timeouts.py'` | 3 passed. LB 10 с > запрос 8 с > acquire+SQL 3 с; drain 2 с > обнаружение 1,5 с; shutdown 6 с < hook 7 с < grace 8 с. Настоящий gunicorn: ready/live 503/200, принятый запрос 201, зависший 503 с обоими заголовками; тестовое окно 1 с, порт закрыт через 1,05 с, выход через 2,44 с | `evidence/grade4-drain.txt` | +| **4.1** Обнаружение отказа | `make detection` | 12/12 PASS; SIGTERM 1,069–1,266 с, kill 0,886–0,934 с, pause 2,778–3,215 с, PostgreSQL 1,196–1,263 с. Все в пределах формулы без допуска; SIGTERM исключён до конца остановки | `evidence/grade4-detection.txt`, `evidence/detection/` | +| **4.2** Каждая реплика под нагрузкой | `make failover-each RATE=100` | Два прогона, по 6001 запросу, 0 ошибок и dropped; конкретная реплика Exited (0), сосед healthy; исключённая реплика не обслуживает зонд, после restore возвращается; история полностью совпадает | `evidence/grade4-failover-each.txt`, `evidence/failover-each//action.txt`, `result.json`, `probe.csv`, `lb_up.csv`, `k6/` | +| **4.2** Общая идемпотентность | `make lb-check` | 20 × 8 параллельных POST: 160 × 201, по 80 от каждой реплики; 20/20 одинаковых тел, единственных копий и конфликтов другого body | `evidence/grade4-baseline.txt`, `evidence/lb-check.json` | +| **4.3** Лимиты и изоляция | `make limits`; `docker inspect` во время k6 | CPU/MiB: API 1/512 на реплику, LB 1/128, PostgreSQL 2/1024, Valkey 0,5/128, migrate 0,5/256, k6 2/512. Лимиты реально применены; только LB публикует порты на loopback, API read_only и без container_name; secrets/volumes/сети соответствуют политике | `evidence/limits.txt`, `evidence/grade4-runtime-limits.txt` | +| **4.3** Отрицательная проверка политики | `scripts/limits.py` с override из лога | Все 5 нарушений обнаружены, ожидаемый exit 1; конфигурация не применялась к стенду | `evidence/grade4-limits-negative.txt`, `evidence/limits-violations.txt` | +| **4.4** Поочерёдная замена | `make rolling-update RATE=100`; `make rolling-update FROM_REV=7fc911c RATE=100` | В обоих прогонах минимум 2 UP, по 9001 запросу, 0 ошибок/dropped, 913/913 сообщений зонда сохранены; после действия отвечают только новые контейнеры. Старые/новые image ID записаны для каждой замены | `evidence/grade4-rolling*.txt`, `evidence/rolling/`, `evidence/rolling-from-7fc911c/` | +| **4.4** Один migrate и совместимость | `scripts/rolling-update.sh`; повтор `make migrate` | В каждом action.txt один migrate перед первой заменой. Обе версии SQLAlchemy используют ту же схему Alembic; повтор upgrade head не меняет версию и счётчики таблиц | `evidence/rolling/action.txt`, `evidence/rolling-from-7fc911c/action.txt`, `evidence/grade4-migration-repeat.txt` | + +Измерения поочерёдного отказа: + +| Остановленный hostname | RPS | p50/p95/p99, мс | Обнаружение, с | Первый успешный ответ, с | Возврат в LB от начала действия, с | Подтверждено = в истории | +| --- | --- | --- | --- | --- | --- | --- | +| `7cfd91ddf780` | 100.02 | 2.97/3.86/4.47 | 0.884 | 0.023 | 14.885 | 645 = 645 | +| `5bbf259ff136` | 100.02 | 2.98/3.88/4.53 | 1.498 | 0.111 | 14.990 | 642 = 642 | + +Возврат включает намеренные 10 с без реплики и её запуск; это не перерыв доступности записи. +Счётчики LB `wretr/wredis/econ/eresp/hrsp_5xx` равны нулю в обоих отказах и обоих rolling update. +Опрос LB под нагрузкой — каждые 100 мс; более короткий провал таким измерением не исключается. + +| Rolling update | RPS | p50/p95/p99, мс | Действие, с | Первый успешный ответ, с | Минимум UP | +| --- | --- | --- | --- | --- | --- | +| Текущая сборка | 100.01 | 3.01/3.89/4.53 | 10.734 | 0.055 | 2 | +| `7fc911c` → текущая | 100.01 | 2.99/3.85/4.47 | 13.836 | 0.082 | 2 | + +**Ограничение миграционного доказательства:** `7fc911c` — предыдущая SQLAlchemy-версия до исправления +восстановления пула, с той же схемой `0002_idempotency_keys`. Миграция в rolling update — no-op. +Это подтверждает однократный запуск и совместную работу версий на существующей схеме, но не произвольный DDL +под нагрузкой. Будущие миграции требуют expand/contract и отдельной проверки совместимости. +Подготовительная установка старого образа выполняется до запуска нагрузки; непрерывность гарантируется +измеренным окном поочерёдной замены. Новый image ID той же сборки может отличаться из-за аттестации BuildKit. + +Публичные сценарии после замены, итоговые проверки и остановка — `evidence/grade4-public.txt`, +`grade4-checks.txt`, `grade4-student-tests.txt`, `grade4-stop.txt`. +Повтор persistence в `grade4-persistence-scoped.txt` фиксирует неизменные IDs и обновлённые StartedAt +всех шести контейнеров. Полный набор собственных тестов — 285 passed, 12 deselected (stack). +Неуспешный smoke через системный прокси, прерванный detection, первоначальные ошибки форматирования +и некорректная отрицательная Compose-fixture сохранены; причины и итоговые повторы перечислены +в `evidence/README.md`. Эти попытки не считаются PASS. **Трек 1 «Цена масштабирования» — исторические результаты до SQLAlchemy.** Подробности, таблицы и выводы — в разделе трека ниже; поля CSV — `evidence/track1/README.md`. @@ -560,8 +591,8 @@ CPU и вдвое больший p95 ниже колена. Значит, рет но пул по-прежнему занят ждущими блокировку: теперь держатель ждёт не приложение, а сброс WAL при фиксации (ожидания `LWLock`, почти все — `WALWrite`, 8,5–9,2 против 1,9–2,3 «до»; `pg_activity.csv`), а CPU PostgreSQL — 89 %. Нижние значения варианта «после» за коленом (4 494 и 4 618 rps) — как раз возмущённые прогоны (см. -методику): вывод они не усиливают, а ослабляют. Проверки уровней «3» и «4» на новом коде проходят -(`after-checks.txt`, `clean-clone.txt`), а на коде до SQLAlchemy повторён весь уровень «4» под k6 (строки 4.x). +методику): вывод они не усиливают, а ослабляют. Исторические проверки уровней «3» и «4» после этого улучшения сохранены +в `after-checks.txt` и `clean-clone.txt`. Актуальные проверки SQLAlchemy перечислены в строках 4.x выше. **Выводы.** @@ -615,7 +646,7 @@ CPU и вдвое больший p95 ниже колена. Значит, рет **Чем отказ процесса отличается от отказа узла?** Отказ процесса — это одна реплика: контейнер остановлен, убит или завис, а host, LB, PostgreSQL и соседняя реплика работают. Его ловит проверка готовности LB, и -трафик уходит на соседа: `SIGTERM` — 1,3 с, `kill` — 0,4–0,5 с, зависание (`docker pause`) — 2,9–3,1 с по таймауту +трафик уходит на соседа: `SIGTERM` — 1,069–1,266 с, `kill` — 0,886–0,934 с, зависание (`docker pause`) — 2,778–3,215 с по таймауту проверки (`evidence/detection/`), запись не прерывается (`failover-each`: 0 ошибок под 100 rps). Отказ узла забирает всё, что на нём запущено, сразу: здесь узел один, поэтому вместе с репликами пропадают LB, PostgreSQL, Valkey и volumes — отказ всего стенда, и никакая реплика его не переживёт. Кроме того, мёртвый узел не отвечает @@ -631,7 +662,7 @@ Valkey и volumes — отказ всего стенда, и никакая ре повторов отказавшего соединения на соседа, мягкого вывода из ротации и ответа `503` в формате контракта, когда готовых реплик нет. Всё это делает HAProxy: активная проверка `/health/ready` раз в 1 с с `fall 2`/`rise 2`, round robin, `retry-on conn-failure` и `redispatch`. Пример — остановка PostgreSQL: контейнеры `api` продолжают -работать, а LB исключает обе реплики за 1,0–1,3 с и сам отвечает `503 unavailable` (`detection/run*/postgres.json`, `faults.txt`). +работать, а LB исключает обе реплики за 1,196–1,263 с и сам отвечает `503 unavailable` (`detection/run*/postgres.json`). **Какие POST можно повторять после timeout и что защищает Idempotency-Key?** Только `POST /api/v1/chats/{chat_id}/messages` с тем же `Idempotency-Key` и тем же телом. Сообщение и квитанция ключа @@ -661,8 +692,8 @@ persistence с прежним токеном падают. Sticky-сессии ## Ограничения и следующие шаги -Следующий этап — уровень «4», затем повторный трек «Цена масштабирования» на SQLAlchemy. -Числа обнаружения отказов, rolling update и улучшения трека в списке ниже относятся к историческим прогонам. +Следующий этап — повторный трек «Цена масштабирования» на SQLAlchemy. Уровни «3» и «4» подтверждены; +цифры улучшения трека в списке ниже остаются историческими. - Один Docker-host: потеря host, LB или PostgreSQL — отказ всего стенда. LB и БД — единственные точки отказа; их резервирование вне рамок лабы. @@ -680,15 +711,15 @@ persistence с прежним токеном падают. Sticky-сессии секунду: при большем `inter` окно нужно увеличить (это ловит `test_timeouts.py`). - Время обнаружения измерено тремя прогонами на сценарий; верхняя граница — формула по параметрам проверки. Замер считается от запуска команды отказа и включает задержку docker CLI и период опроса статистики LB, поэтому - может превысить формулу на эту величину; допуск замера — 0,25 с. На коде до SQLAlchemy все замеры в пределах - формулы, раньше превышение в пределах допуска было у PostgreSQL (1,553 с), `SIGTERM` (1,609 и 1,515 с). + может превысить формулу на эту величину; допуск замера — 0,25 с. На текущем коде все 12 замеров в пределах + формулы без использования допуска (`evidence/grade4-detection.txt`). - Rolling update проверен со сменой кода без смены схемы. Миграция, несовместимая с работающей версией (удаление или переименование столбца), потребовала бы двух выпусков expand/contract; в лабе таких нет. - `scripts/rolling-update.sh` и `restore.sh` используют `--no-recreate`: после замены с другим `API_IMAGE` обычный `make up` пересоздаст реплики под текущий конфиг — это ожидаемо. - Секреты лежат файлами в `secrets/` на хосте (каталог `0700`, файлы `0644`, чтобы их читали непривилегированные пользователи контейнеров); это учебное упрощение вместо менеджера секретов. -- Трек 1: подтверждено одно улучшение (запись одной командой, +15 % пика, p95 в 17 раз ниже колена); гипотезы +- Исторический трек 1 до SQLAlchemy: подтверждено одно улучшение (запись одной командой, +15 % пика, p95 в 17 раз ниже колена); гипотезы о троттлинге CFS как причине дорогих мелких реплик и о росте с числом чатов не проверялись. Переход отправки на одну команду убрал агрегат `Chat` из записи: правила участия и порядка теперь в SQL адаптера и держатся интеграционными тестами. diff --git a/evidence/README.md b/evidence/README.md index 2dde3787..a2be57d0 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -2,8 +2,9 @@ Обезличенные реальные прогоны; пользователи, сообщения и пароли тестов синтетические. Токены, содержимое secrets и дампы не сохраняются. В первой строке текстовых логов — UTC и SHA; -`grade3-*.txt` также содержат команду, окружение, код завершения и длительность. -В сохранённом текстовом выводе удалены только пробелы на концах строк Docker/psql; локальный путь заменён на ``. +`grade3-*.txt` и `grade4-*.txt` также содержат команду, окружение, код завершения и длительность. +В текстовом выводе удалены пробелы на концах строк Docker/psql; локальный путь заменён на ``. +В grade4-student-tests.txt замаскирован синтетический неверный Bearer из имени параметризованного теста. ## Уровень «3» после SQLAlchemy — 2026-11-05 @@ -11,7 +12,7 @@ Compose-проект `mtusi2026lab3-grade3-final-20261105`, `LB_HTTP_PORT=18083`, `LB_STATS_PORT=23083`, `API_REPLICAS=2`; BASE_URL автоматически определяется по порту LB. Обе реплики: api-1 `8ffe0c70f4ea`, api-2 `1b4a73f61b07`. После прогонов контейнеры остановлены, -volumes сохранены. Код приложения после этого SHA не менялся; далее — документация и архив OpenSpec. +volumes сохранены. Код приложения после этого SHA не менялся; на этапе grade4 усилена обвязка `scripts/probe.py`. | Файл | Команда | Результат | | --- | --- | --- | @@ -19,7 +20,7 @@ volumes сохранены. Код приложения после этого SH | `checks.txt` | `make check`, `make validate`, `make lint`, `uv lock --check`, `openspec validate --all --strict`, diff против upstream | Все успешно; нормативные файлы не менялись | | `smoke.txt` | `make test` | Публичный smoke PASS | | `stack.txt`, `environment.txt`, `grade3-stack.txt` | `make stack-evidence` | Порты только у LB, реальные контейнеры и hostname, 2 UP, общие PostgreSQL/Valkey, uid, версии библиотек | -| `lb-check.json`, `grade3-lb-check.txt` | `make lb-check` | Один токен 20 + 20 GET; 20 × 8 POST, 160 ответов 201, по одной копии, 20 конфликтов другого body; pass | +| `grade3-lb-check.txt` (актуальный `lb-check.json` — повтор grade4) | `make lb-check` | Один токен 20 + 20 GET; 20 × 8 POST, 160 ответов 201, по одной копии, 20 конфликтов другого body; pass | | `retry.json`, `grade3-retry.txt` | `make retry-check` | Потерянный ответ при живой/убитой реплике, повтор через соседа 201, одна копия; pass | | `failover-current-1.txt`, `failover-current-2.txt` | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` и `docker compose ps -a` | По очереди api-1 и api-2 Exited (0); 129/128 записей, 0/0 ошибок, восстановление 2.762/2.817 с | | `grade3-restore-1.txt`, `grade3-restore-2.txt` | `make restore` | Остановленные реплики возвращены в ротацию | @@ -31,8 +32,8 @@ volumes сохранены. Код приложения после этого SH | `grade3-final-review.txt` | Финальные проверки после обновления отчёта и архива | check/validate/lint/OpenSpec, diff нормативных и исполняемых файлов, отсутствие контейнеров этапа | | `grade3-stop.txt` | `make down`, `docker compose ps -a`, volumes проектов | Контейнеров нет, volumes обоих прогонов сохранены | -Дата и полный SHA JSON-файлов `lb-check.json` и `retry.json` находятся в соответствующих -`grade3-lb-check.txt` и `grade3-retry.txt`, которые содержат тот же вывод JSON. +Исходный JSON уровня «3» сохранён в `grade3-lb-check.txt` и `grade3-retry.txt` вместе с датой и полным SHA. +`lb-check.json` обновлён повторным прогоном grade4; `retry.json` остаётся результатом grade3. ## Неуспешные промежуточные прогоны @@ -49,21 +50,48 @@ volumes сохранены. Код приложения после этого SH | `grade3-initial-stop.txt` | `18de595` + WIP | Остановка исходного проекта без удаления его volumes перед итоговым прогоном | `clean-clone.txt`, `hooks.txt`, `failover-fe75c6c37e5a.txt`, `failover-ff81f93e8b0d.txt` -и данные уровня «4»/трека ниже — **история до SQLAlchemy**, с собственными датами и SHA. -Исторические версии перезаписанных файлов доступны через `git show 7fc911c:evidence/<имя>`. -Они не подтверждают актуальную производительность. Уровень «4», затем трек «Цена масштабирования» -перегенерируются следующими последовательными этапами; исходный change остаётся активным. +и `rolling-from-8d28d45/` — история до SQLAlchemy. Прежние версии перезаписанных доказательств +доступны через `git show ef0b9a3:evidence/<имя>`. Трек ниже ещё предстоит перегенерировать; +исходный SQLAlchemy change остаётся активным. -## Уровень «4» — исторические результаты до SQLAlchemy +## Уровень «4» после SQLAlchemy — 2026-11-05 -| Файл | Что доказывает | Команда | Коммит | -| --- | --- | --- | --- | -| `limits.txt` | лимиты CPU/RAM всех сервисов (объявлены и применены), порты только у LB на `127.0.0.1`, секреты, volumes и сети по политике | `make limits` | `e049c26` | -| `limits-violations.txt` | та же проверка ловит подмешанные нарушения (5 штук, код 1) | `COMPOSE_FILE=compose.yaml:violations.yaml python3 scripts/limits.py` | `e049c26` | -| `detection/run{1,2,3}/{sigterm,kill,pause,postgres}.json` | три прогона: время исключения backend из ротации против заявленного максимума (допуск замера 0,25 с), ответ LB после исключения, время возврата | `make detection` | `e049c26` | -| `failover-each/summary.json`, `failover-each//` | поочерёдная остановка каждой реплики под k6 100 rps: RPS, ошибки, p50/p95/p99, обнаружение, возврат, зонд записи, счётчики LB; сырьё — `k6/`, `lb_up.csv`, `probe.csv`, `action.txt` (вывод действия: `docker compose ps -a` реплик сразу после хука, размер БД и `docker stats` стенда без остановленной реплики, затем `restore`) | `make failover-each` | `e049c26` | -| `rolling-from-8d28d45/` | rolling update с образа уровня «3» (`mtusi-messenger:8d28d45`) на текущий под k6: ёмкость, ошибки, замена всех реплик, `migrate` один раз; в `action.txt` — тег и ID образа каждой заменяемой и новой реплики, размер БД и `docker stats` | `make rolling-update FROM_REV=8d28d45` | `e049c26` | -| `rolling/` | rolling update текущей сборкой под k6 | `make rolling-update` | `e049c26` | +Код прогонов: **`252e3e3ffeea5004cfd9dc342e686af1417a77a2`**. +Проект `mtusi2026lab3-grade4-20261105`, порты `18083/23083`, две реплики по 1 CPU/512 MiB, +два воркера и пул 8 на воркер; HAProxy maxconn 64. Локальный HTTP явно обходит системный прокси +через `NO_PROXY=localhost,127.0.0.1,::1`. Команды и исходные результаты сохранены ниже. +JSON содержит поля date/commit; первая строка CSV — служебный комментарий с UTC/SHA, +который следует пропустить при чтении таблицы. Эти метаданные добавлены после прогонов; +результаты не изменялись. Из текстового вывода убраны пробелы на концах строк. + +| Файл | Команда | Результат | +| --- | --- | --- | +| `grade4-start.txt` | `make up`, версии пакетов, `docker compose ps -a`, Alembic head | Новый проект с пустыми volumes; SQLAlchemy 2.0.45 / Alembic 1.16.5; migrate Exited (0), head `0002_idempotency_keys` | +| `grade4-baseline.txt`, `lb-check.json`, `limits.txt` | `make test`, `make limits`, `make lb-check` | Smoke PASS; порты только LB; 160 ответов 201 через обе реплики, 20/20 одинаковых тел/единственных копий/конфликтов другого body | +| `grade4-drain.txt` | `make student-test PYTEST_ARGS='-v -s student-tests/api/test_drain.py student-tests/unit/test_timeouts.py'` | 3 passed; ready/live 503/200; порт закрыт через 1.05 с, выход через 2.44 с; принятый запрос → 201, зависший → 503 с заголовками контракта | +| `grade4-detection.txt`, `detection/run{1,2,3}/*.json` | `make detection` | Все 12 PASS, диапазоны и пределы — в REPORT | +| `grade4-failover-each.txt`, `failover-each/` | `make failover-each RATE=100` | Каждая реплика Exited (0) и возвращена; по 6001 запросу, 0 ошибок/dropped; 645/642 сообщения зонда полностью сохранены | +| `grade4-runtime-limits.txt` | `docker ps -a` и `docker inspect` во время k6 | Фактические CPU/RAM, uid, image ID, порты и read_only всех сервисов, включая k6 и завершённый migrate | +| `grade4-limits-negative.txt`, `limits-violations.txt` | `scripts/limits.py` с временным override из лога, `API_REPLICAS=1` только для config | Ожидаемый exit 1, все 5 нарушений найдены; override никогда не применялся к контейнерам | +| `grade4-rolling.txt`, `rolling/` | `make rolling-update RATE=100` | Новые реплики готовы до остановки прежних; один migrate, минимум 2 UP, все checks PASS | +| `grade4-rolling-from-7fc911c.txt`, `rolling-from-7fc911c/` | `make rolling-update FROM_REV=7fc911c RATE=100` | Переход между SQLAlchemy-версиями; старый/новый image ID каждой реплики в action.txt | +| `grade4-migration-repeat.txt` | Повторный `make migrate`, Alembic head и счётчики всех таблиц до/после | Результат записан отдельно; миграция выполняется при остановленной нагрузке | +| `grade4-persistence-scoped.txt` | Публичный persistence и docker inspect до/после | IDs контейнеров прежние, StartedAt всех lb/api/postgres/valkey/migrate обновлены | +| `grade4-public.txt` | Публичные smoke/persistence/failover, restore и реальные контейнеры | Итоговые проверки после rolling update | +| `grade4-evidence-audit.txt` | Сверка result.json с сырыми CSV, проверка количества миграторов, поиск похожих на credentials строк | Все 12 detection и 4 нагрузки PASS; численные поля совпадают; один migrate на rolling | +| `grade4-checks.txt`, `grade4-student-tests.txt` | check/validate/lint/lock/OpenSpec, полный student-test | Итоговые проверки на коде stage4 | +| `grade4-stop.txt` | `make down`, список контейнеров и volumes проекта | Контейнеры остановлены, volumes сохранены | + +Промежуточные неуспехи не заменяют итоговые результаты: + +| Файл | Наблюдение | +| --- | --- | +| `grade4-evidence-audit-scan-failed.txt`, `grade4-evidence-audit-fixture-token.txt` | Численные сверки PASS. Первый поиск нашёл собственный regex; после уточнения — синтетический неверный Bearer в имени теста. Имя параметра замаскировано, итоговый поиск прошёл | +| `grade4-development.txt` | Первоначальные E501/ARG005, исправления форматирования, 14 новых регрессионных тестов; уточнение library ID Context7 | +| `grade4-baseline-proxy-failed.txt` | Smoke: RemoteDisconnected из-за системного прокси; прямой curl отвечал 200; повтор с NO_PROXY прошёл | +| `grade4-detection-proxy-interrupted.txt`, `grade4-detection-proxy-interrupted/` | Прогон без NO_PROXY: статистика LB недоступна, SIGTERM честно fail; остановлен во время kill, реплики восстановлены; выполнен полный новый прогон | +| `grade4-public-events-unavailable.txt` | Дополнительная ретроспективная проверка через docker events не подтвердила рестарт: в оставшемся буфере не было событий start/die нужного окна. Persistence повторён с непосредственным снимком StartedAt до/после | +| `grade4-limits-negative-config-failed.txt`, `grade4-limits-compose-rejection.txt` | Compose отклонил сочетание container_name и replicas=2 до проверки политики; отрицательная fixture повторена с replicas=1 (конфигурация не применялась) | ## Трек 1 «Цена масштабирования» — исторические результаты до SQLAlchemy diff --git a/evidence/detection/run1/kill.json b/evidence/detection/run1/kill.json index 9277e399..468bf5cc 100644 --- a/evidence/detection/run1/kill.json +++ b/evidence/detection/run1/kill.json @@ -1,14 +1,15 @@ -{ +{"date": "2026-11-05T19:17:01.673107+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "action": [ "docker", "kill", - "ff81f93e8b0d3eba1c06f5017cbe7e44747a32708857ccf83f613c06eea429f3" + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" ], "action_exit": 0, - "action_seconds": 0.119, + "action_seconds": 0.113, "lb_up_before": 2, + "lb_samples_missing": 0, "lb_up_counted_as_detected": 1, - "detection_seconds": 0.429, + "detection_seconds": 0.934, "declared_max_seconds": 1.5, "measurement_slack_seconds": 0.25, "lb_answer_after_detection": { @@ -18,10 +19,11 @@ "error_code": null }, "restore": "scripts/restore.sh", - "restore_seconds": 3.006, + "restore_seconds": 2.57, "lb_up_after_restore": 2, "checks": { "action_ok": true, + "lb_observed": true, "detected_in_time": true, "lb_answer_ok": true, "restored": true diff --git a/evidence/detection/run1/pause.json b/evidence/detection/run1/pause.json index 2285b984..2c60437f 100644 --- a/evidence/detection/run1/pause.json +++ b/evidence/detection/run1/pause.json @@ -1,14 +1,15 @@ -{ +{"date": "2026-11-05T19:17:01.673107+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "action": [ "docker", "pause", - "ff81f93e8b0d3eba1c06f5017cbe7e44747a32708857ccf83f613c06eea429f3" + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" ], "action_exit": 0, - "action_seconds": 0.063, + "action_seconds": 0.057, "lb_up_before": 2, + "lb_samples_missing": 0, "lb_up_counted_as_detected": 1, - "detection_seconds": 2.926, + "detection_seconds": 3.215, "declared_max_seconds": 3.5, "measurement_slack_seconds": 0.25, "lb_answer_after_detection": { @@ -17,11 +18,12 @@ "x_instance_id_present": true, "error_code": null }, - "restore": "docker unpause ff81f93e8b0d3eba1c06f5017cbe7e44747a32708857ccf83f613c06eea429f3 && scripts/lb-wait.sh", - "restore_seconds": 2.365, + "restore": "docker unpause 7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 && scripts/lb-wait.sh", + "restore_seconds": 2.425, "lb_up_after_restore": 2, "checks": { "action_ok": true, + "lb_observed": true, "detected_in_time": true, "lb_answer_ok": true, "restored": true diff --git a/evidence/detection/run1/postgres.json b/evidence/detection/run1/postgres.json index 9a58bd60..566c0ed8 100644 --- a/evidence/detection/run1/postgres.json +++ b/evidence/detection/run1/postgres.json @@ -1,14 +1,15 @@ -{ +{"date": "2026-11-05T19:17:01.673107+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "action": [ "docker", "stop", - "cd7b0d78bc19e016ceffb81b83e0286012fc8bf779ae2724c52f5477ee04e19f" + "f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a" ], "action_exit": 0, - "action_seconds": 0.121, + "action_seconds": 0.167, "lb_up_before": 2, + "lb_samples_missing": 0, "lb_up_counted_as_detected": 0, - "detection_seconds": 1.266, + "detection_seconds": 1.196, "declared_max_seconds": 1.5, "measurement_slack_seconds": 0.25, "lb_answer_after_detection": { @@ -17,11 +18,12 @@ "x_instance_id_present": false, "error_code": "unavailable" }, - "restore": "docker start cd7b0d78bc19e016ceffb81b83e0286012fc8bf779ae2724c52f5477ee04e19f && scripts/lb-wait.sh", + "restore": "docker start f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a && scripts/lb-wait.sh", "restore_seconds": 2.475, "lb_up_after_restore": 2, "checks": { "action_ok": true, + "lb_observed": true, "detected_in_time": true, "lb_answer_ok": true, "restored": true diff --git a/evidence/detection/run1/sigterm.json b/evidence/detection/run1/sigterm.json index 31b5abd3..fad94e44 100644 --- a/evidence/detection/run1/sigterm.json +++ b/evidence/detection/run1/sigterm.json @@ -1,16 +1,17 @@ -{ +{"date": "2026-11-05T19:17:01.673107+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "action": [ "docker", "stop", "-t", "7", - "ff81f93e8b0d3eba1c06f5017cbe7e44747a32708857ccf83f613c06eea429f3" + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" ], "action_exit": 0, - "action_seconds": 2.539, + "action_seconds": 2.535, "lb_up_before": 2, + "lb_samples_missing": 0, "lb_up_counted_as_detected": 1, - "detection_seconds": 1.259, + "detection_seconds": 1.069, "declared_max_seconds": 1.5, "measurement_slack_seconds": 0.25, "lb_answer_after_detection": { @@ -20,10 +21,11 @@ "error_code": null }, "restore": "scripts/restore.sh", - "restore_seconds": 2.514, + "restore_seconds": 1.969, "lb_up_after_restore": 2, "checks": { "action_ok": true, + "lb_observed": true, "detected_in_time": true, "lb_answer_ok": true, "excluded_before_action_end": true, diff --git a/evidence/detection/run2/kill.json b/evidence/detection/run2/kill.json index 086030d4..e8115f3d 100644 --- a/evidence/detection/run2/kill.json +++ b/evidence/detection/run2/kill.json @@ -1,14 +1,15 @@ -{ +{"date": "2026-11-05T19:17:01.673107+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "action": [ "docker", "kill", - "ff81f93e8b0d3eba1c06f5017cbe7e44747a32708857ccf83f613c06eea429f3" + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" ], "action_exit": 0, - "action_seconds": 0.173, + "action_seconds": 0.162, "lb_up_before": 2, + "lb_samples_missing": 0, "lb_up_counted_as_detected": 1, - "detection_seconds": 0.526, + "detection_seconds": 0.901, "declared_max_seconds": 1.5, "measurement_slack_seconds": 0.25, "lb_answer_after_detection": { @@ -18,10 +19,11 @@ "error_code": null }, "restore": "scripts/restore.sh", - "restore_seconds": 2.985, + "restore_seconds": 3.086, "lb_up_after_restore": 2, "checks": { "action_ok": true, + "lb_observed": true, "detected_in_time": true, "lb_answer_ok": true, "restored": true diff --git a/evidence/detection/run2/pause.json b/evidence/detection/run2/pause.json index 19a89fbc..29b23edf 100644 --- a/evidence/detection/run2/pause.json +++ b/evidence/detection/run2/pause.json @@ -1,14 +1,15 @@ -{ +{"date": "2026-11-05T19:17:01.673107+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "action": [ "docker", "pause", - "ff81f93e8b0d3eba1c06f5017cbe7e44747a32708857ccf83f613c06eea429f3" + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" ], "action_exit": 0, "action_seconds": 0.063, "lb_up_before": 2, + "lb_samples_missing": 0, "lb_up_counted_as_detected": 1, - "detection_seconds": 3.063, + "detection_seconds": 2.817, "declared_max_seconds": 3.5, "measurement_slack_seconds": 0.25, "lb_answer_after_detection": { @@ -17,11 +18,12 @@ "x_instance_id_present": true, "error_code": null }, - "restore": "docker unpause ff81f93e8b0d3eba1c06f5017cbe7e44747a32708857ccf83f613c06eea429f3 && scripts/lb-wait.sh", - "restore_seconds": 2.353, + "restore": "docker unpause 7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 && scripts/lb-wait.sh", + "restore_seconds": 2.378, "lb_up_after_restore": 2, "checks": { "action_ok": true, + "lb_observed": true, "detected_in_time": true, "lb_answer_ok": true, "restored": true diff --git a/evidence/detection/run2/postgres.json b/evidence/detection/run2/postgres.json index 618e6452..6795c2bd 100644 --- a/evidence/detection/run2/postgres.json +++ b/evidence/detection/run2/postgres.json @@ -1,12 +1,13 @@ -{ +{"date": "2026-11-05T19:17:01.673107+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "action": [ "docker", "stop", - "cd7b0d78bc19e016ceffb81b83e0286012fc8bf779ae2724c52f5477ee04e19f" + "f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a" ], "action_exit": 0, - "action_seconds": 0.171, + "action_seconds": 0.17, "lb_up_before": 2, + "lb_samples_missing": 0, "lb_up_counted_as_detected": 0, "detection_seconds": 1.263, "declared_max_seconds": 1.5, @@ -17,11 +18,12 @@ "x_instance_id_present": false, "error_code": "unavailable" }, - "restore": "docker start cd7b0d78bc19e016ceffb81b83e0286012fc8bf779ae2724c52f5477ee04e19f && scripts/lb-wait.sh", - "restore_seconds": 2.417, + "restore": "docker start f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a && scripts/lb-wait.sh", + "restore_seconds": 2.455, "lb_up_after_restore": 2, "checks": { "action_ok": true, + "lb_observed": true, "detected_in_time": true, "lb_answer_ok": true, "restored": true diff --git a/evidence/detection/run2/sigterm.json b/evidence/detection/run2/sigterm.json index b111e058..0f8cffd3 100644 --- a/evidence/detection/run2/sigterm.json +++ b/evidence/detection/run2/sigterm.json @@ -1,16 +1,17 @@ -{ +{"date": "2026-11-05T19:17:01.673107+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "action": [ "docker", "stop", "-t", "7", - "ff81f93e8b0d3eba1c06f5017cbe7e44747a32708857ccf83f613c06eea429f3" + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" ], "action_exit": 0, - "action_seconds": 2.471, + "action_seconds": 2.662, "lb_up_before": 2, + "lb_samples_missing": 0, "lb_up_counted_as_detected": 1, - "detection_seconds": 1.348, + "detection_seconds": 1.205, "declared_max_seconds": 1.5, "measurement_slack_seconds": 0.25, "lb_answer_after_detection": { @@ -20,10 +21,11 @@ "error_code": null }, "restore": "scripts/restore.sh", - "restore_seconds": 3.555, + "restore_seconds": 1.953, "lb_up_after_restore": 2, "checks": { "action_ok": true, + "lb_observed": true, "detected_in_time": true, "lb_answer_ok": true, "excluded_before_action_end": true, diff --git a/evidence/detection/run3/kill.json b/evidence/detection/run3/kill.json index ef0ec1f7..47bb840e 100644 --- a/evidence/detection/run3/kill.json +++ b/evidence/detection/run3/kill.json @@ -1,14 +1,15 @@ -{ +{"date": "2026-11-05T19:17:01.673107+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "action": [ "docker", "kill", - "ff81f93e8b0d3eba1c06f5017cbe7e44747a32708857ccf83f613c06eea429f3" + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" ], "action_exit": 0, - "action_seconds": 0.119, + "action_seconds": 0.163, "lb_up_before": 2, + "lb_samples_missing": 0, "lb_up_counted_as_detected": 1, - "detection_seconds": 0.42, + "detection_seconds": 0.886, "declared_max_seconds": 1.5, "measurement_slack_seconds": 0.25, "lb_answer_after_detection": { @@ -18,10 +19,11 @@ "error_code": null }, "restore": "scripts/restore.sh", - "restore_seconds": 3.037, + "restore_seconds": 3.095, "lb_up_after_restore": 2, "checks": { "action_ok": true, + "lb_observed": true, "detected_in_time": true, "lb_answer_ok": true, "restored": true diff --git a/evidence/detection/run3/pause.json b/evidence/detection/run3/pause.json index 45ba98f9..9b506a41 100644 --- a/evidence/detection/run3/pause.json +++ b/evidence/detection/run3/pause.json @@ -1,14 +1,15 @@ -{ +{"date": "2026-11-05T19:17:01.673107+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "action": [ "docker", "pause", - "ff81f93e8b0d3eba1c06f5017cbe7e44747a32708857ccf83f613c06eea429f3" + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" ], "action_exit": 0, - "action_seconds": 0.056, + "action_seconds": 0.063, "lb_up_before": 2, + "lb_samples_missing": 0, "lb_up_counted_as_detected": 1, - "detection_seconds": 2.981, + "detection_seconds": 2.778, "declared_max_seconds": 3.5, "measurement_slack_seconds": 0.25, "lb_answer_after_detection": { @@ -17,11 +18,12 @@ "x_instance_id_present": true, "error_code": null }, - "restore": "docker unpause ff81f93e8b0d3eba1c06f5017cbe7e44747a32708857ccf83f613c06eea429f3 && scripts/lb-wait.sh", - "restore_seconds": 2.347, + "restore": "docker unpause 7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 && scripts/lb-wait.sh", + "restore_seconds": 2.416, "lb_up_after_restore": 2, "checks": { "action_ok": true, + "lb_observed": true, "detected_in_time": true, "lb_answer_ok": true, "restored": true diff --git a/evidence/detection/run3/postgres.json b/evidence/detection/run3/postgres.json index 76467251..d3707d3b 100644 --- a/evidence/detection/run3/postgres.json +++ b/evidence/detection/run3/postgres.json @@ -1,14 +1,15 @@ -{ +{"date": "2026-11-05T19:17:01.673107+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "action": [ "docker", "stop", - "cd7b0d78bc19e016ceffb81b83e0286012fc8bf779ae2724c52f5477ee04e19f" + "f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a" ], "action_exit": 0, - "action_seconds": 0.124, + "action_seconds": 0.163, "lb_up_before": 2, + "lb_samples_missing": 0, "lb_up_counted_as_detected": 0, - "detection_seconds": 1.264, + "detection_seconds": 1.239, "declared_max_seconds": 1.5, "measurement_slack_seconds": 0.25, "lb_answer_after_detection": { @@ -17,11 +18,12 @@ "x_instance_id_present": false, "error_code": "unavailable" }, - "restore": "docker start cd7b0d78bc19e016ceffb81b83e0286012fc8bf779ae2724c52f5477ee04e19f && scripts/lb-wait.sh", - "restore_seconds": 2.444, + "restore": "docker start f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a && scripts/lb-wait.sh", + "restore_seconds": 2.481, "lb_up_after_restore": 2, "checks": { "action_ok": true, + "lb_observed": true, "detected_in_time": true, "lb_answer_ok": true, "restored": true diff --git a/evidence/detection/run3/sigterm.json b/evidence/detection/run3/sigterm.json index 34dc60e5..051dfcfe 100644 --- a/evidence/detection/run3/sigterm.json +++ b/evidence/detection/run3/sigterm.json @@ -1,16 +1,17 @@ -{ +{"date": "2026-11-05T19:17:01.673107+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "action": [ "docker", "stop", "-t", "7", - "ff81f93e8b0d3eba1c06f5017cbe7e44747a32708857ccf83f613c06eea429f3" + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" ], "action_exit": 0, - "action_seconds": 2.555, + "action_seconds": 2.665, "lb_up_before": 2, + "lb_samples_missing": 0, "lb_up_counted_as_detected": 1, - "detection_seconds": 1.289, + "detection_seconds": 1.266, "declared_max_seconds": 1.5, "measurement_slack_seconds": 0.25, "lb_answer_after_detection": { @@ -20,10 +21,11 @@ "error_code": null }, "restore": "scripts/restore.sh", - "restore_seconds": 3.552, + "restore_seconds": 1.984, "lb_up_after_restore": 2, "checks": { "action_ok": true, + "lb_observed": true, "detected_in_time": true, "lb_answer_ok": true, "excluded_before_action_end": true, diff --git a/evidence/failover-each/5bbf259ff136/action.txt b/evidence/failover-each/5bbf259ff136/action.txt new file mode 100644 index 00000000..ac29eb30 --- /dev/null +++ b/evidence/failover-each/5bbf259ff136/action.txt @@ -0,0 +1,15 @@ +date: 2026-11-05T19:18:13.470607+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +7cfd91ddf780 api Up About a minute (healthy) +5bbf259ff136 api Exited (0) Less than a second ago +19:19:56 database bytes,messages: 12678835,8785 +mtusi2026lab3-grade4-20261105-api-1 cpu=25.25% mem=209.7MiB / 512MiB +mtusi2026lab3-grade4-20261105-lb-1 cpu=2.71% mem=30.9MiB / 128MiB +mtusi2026lab3-grade4-20261105-postgres-1 cpu=6.45% mem=51.05MiB / 1GiB +mtusi2026lab3-grade4-20261105-valkey-1 cpu=0.43% mem=12.96MiB / 128MiB + Container mtusi2026lab3-grade4-20261105-api-1 Running + Container mtusi2026lab3-grade4-20261105-api-2 Starting + Container mtusi2026lab3-grade4-20261105-api-2 Started + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy diff --git a/evidence/failover-each/5bbf259ff136/k6/k6.txt b/evidence/failover-each/5bbf259ff136/k6/k6.txt new file mode 100644 index 00000000..98f3f03c --- /dev/null +++ b/evidence/failover-each/5bbf259ff136/k6/k6.txt @@ -0,0 +1,3 @@ +date: 2026-11-05T19:18:13.470607+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +time="2026-11-05T19:19:28Z" level=info msg=LOAD-START source=console +measure: 6001 requests, failed ratio 0 diff --git a/evidence/failover-each/5bbf259ff136/k6/summary.json b/evidence/failover-each/5bbf259ff136/k6/summary.json new file mode 100644 index 00000000..b1a9c434 --- /dev/null +++ b/evidence/failover-each/5bbf259ff136/k6/summary.json @@ -0,0 +1,309 @@ +{"date": "2026-11-05T19:18:13.470607+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "p(90)", + "p(95)", + "p(99)", + "max" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": false, + "isStdErrTTY": false, + "testRunDurationMs": 70512.503422 + }, + "metrics": { + "dropped_iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 0, + "rate": 0 + } + }, + "data_sent": { + "values": { + "count": 2054247, + "rate": 29133.08846384076 + }, + "type": "counter", + "contains": "data" + }, + "http_req_tls_handshaking": { + "contains": "time", + "values": { + "med": 0, + "p(90)": 0, + "p(95)": 0, + "p(99)": 0, + "max": 0, + "avg": 0, + "min": 0 + }, + "type": "trend" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(99)": 5.295193749999997, + "max": 53.791625, + "avg": 3.101954274815648, + "min": 0.861194, + "med": 2.9916875000000003, + "p(90)": 3.6507209, + "p(95)": 3.9084498499999993 + } + }, + "checks": { + "contains": "default", + "values": { + "rate": 1, + "passes": 7002, + "fails": 0 + }, + "type": "rate" + }, + "http_req_duration{scenario:measure}": { + "type": "trend", + "contains": "time", + "values": { + "min": 0.861194, + "med": 2.979916, + "p(90)": 3.638291, + "p(95)": 3.8775, + "p(99)": 4.531834, + "max": 53.791625, + "avg": 3.0515494415930533 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "med": 0.011667, + "p(90)": 0.0155, + "p(95)": 0.018935449999999986, + "p(99)": 0.033915669999999995, + "max": 0.254083, + "avg": 0.01292155005672162, + "min": 0.002584 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 7052 + } + }, + "http_reqs{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 6001, + "rate": 85.10547362196873 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 3.101954274815648, + "min": 0.861194, + "med": 2.9916875000000003, + "p(90)": 3.6507209, + "p(95)": 3.9084498499999993, + "p(99)": 5.295193749999997, + "max": 53.791625 + } + }, + "http_req_receiving": { + "contains": "time", + "values": { + "p(95)": 0.04025, + "p(99)": 0.06647825, + "max": 0.282375, + "avg": 0.023369897192285907, + "min": 0.008, + "med": 0.020459, + "p(90)": 0.033583 + }, + "type": "trend" + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 3.7281128, + "p(95)": 3.97822085, + "p(99)": 4.60467816, + "max": 53.948708, + "avg": 3.1402372599257378, + "min": 1.758833, + "med": 3.0767085 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 1, + "min": 0, + "max": 1 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 7002, + "rate": 99.30153746059408 + } + }, + "dropped_iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 0, + "rate": 0 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 17075371, + "rate": 242160.89588832358 + } + }, + "http_req_duration{scenario:measure,name:post_message}": { + "values": { + "max": 53.791625, + "avg": 2.996232064174453, + "min": 0.861194, + "med": 2.948, + "p(90)": 3.5552918, + "p(95)": 3.7865877, + "p(99)": 4.3190254999999995 + }, + "type": "trend", + "contains": "time" + }, + "http_reqs": { + "values": { + "count": 7052, + "rate": 100.0106315584275 + }, + "type": "counter", + "contains": "default" + }, + "vus_max": { + "contains": "default", + "values": { + "value": 20, + "min": 20, + "max": 20 + }, + "type": "gauge" + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 3.8635248499999997, + "p(99)": 5.2155688399999915, + "max": 53.760833, + "avg": 3.065662827566643, + "min": 0.829445, + "med": 2.9556665, + "p(90)": 3.6086503 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "min": 0.000625, + "med": 0.001667, + "p(90)": 0.002541, + "p(95)": 0.003041, + "p(99)": 0.006434919999999987, + "max": 0.540333, + "avg": 0.002490143788996061 + } + }, + "http_req_connecting": { + "contains": "time", + "values": { + "p(99)": 0, + "max": 0.235959, + "avg": 0.00042762110039705047, + "min": 0, + "med": 0, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend" + }, + "http_req_duration{scenario:measure,name:get_history}": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 4.139021, + "p(99)": 5.169410699999997, + "max": 51.836375, + "avg": 3.2761305311973, + "min": 1.79625, + "med": 3.1579794999999997, + "p(90)": 3.8563755 + } + }, + "http_req_failed{scenario:measure}": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 6001 + } + }, + "iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 6001, + "rate": 85.10547362196873 + } + } + }, + "root_group": { + "groups": [], + "checks": [ + { + "path": "::message 201", + "id": "6445aff337e1ade0bc10402fbac79882", + "passes": 5622, + "fails": 0, + "name": "message 201" + }, + { + "fails": 0, + "name": "history 200", + "path": "::history 200", + "id": "fe27b3b8edd292c7f2170a6357b087d0", + "passes": 1380 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e" + } +} diff --git a/evidence/failover-each/5bbf259ff136/lb_up.csv b/evidence/failover-each/5bbf259ff136/lb_up.csv new file mode 100644 index 00000000..4ed30533 --- /dev/null +++ b/evidence/failover-each/5bbf259ff136/lb_up.csv @@ -0,0 +1,657 @@ +# date: 2026-11-05T19:18:13.470607+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +t_rel_s,up +-25.748,2 +-25.645,2 +-25.541,2 +-25.439,2 +-25.328,2 +-25.218,2 +-25.105,2 +-25.002,2 +-24.89,2 +-24.778,2 +-24.667,2 +-24.556,2 +-24.453,2 +-24.341,2 +-24.236,2 +-24.132,2 +-24.025,2 +-23.912,2 +-23.801,2 +-23.691,2 +-23.584,2 +-23.472,2 +-23.36,2 +-23.249,2 +-23.147,2 +-23.035,2 +-22.922,2 +-22.819,2 +-22.712,2 +-22.6,2 +-22.488,2 +-22.381,2 +-22.27,2 +-22.164,2 +-22.056,2 +-21.947,2 +-21.834,2 +-21.724,2 +-21.621,2 +-21.516,2 +-21.404,2 +-21.3,2 +-21.188,2 +-21.083,2 +-20.976,2 +-20.868,2 +-20.756,2 +-20.652,2 +-20.549,2 +-20.442,2 +-20.333,2 +-20.222,2 +-20.119,2 +-20.005,2 +-19.892,2 +-19.784,2 +-19.675,2 +-19.561,2 +-19.449,2 +-19.337,2 +-19.224,2 +-19.121,2 +-19.015,2 +-18.908,2 +-18.798,2 +-18.689,2 +-18.583,2 +-18.476,2 +-18.362,2 +-18.254,2 +-18.141,2 +-18.032,2 +-17.918,2 +-17.806,2 +-17.701,2 +-17.596,2 +-17.492,2 +-17.388,2 +-17.282,2 +-17.179,2 +-17.073,2 +-16.961,2 +-16.853,2 +-16.742,2 +-16.631,2 +-16.517,2 +-16.413,2 +-16.3,2 +-16.191,2 +-16.085,2 +-15.973,2 +-15.86,2 +-15.752,2 +-15.639,2 +-15.533,2 +-15.425,2 +-15.316,2 +-15.21,2 +-15.107,2 +-15.0,2 +-14.893,2 +-14.783,2 +-14.671,2 +-14.558,2 +-14.448,2 +-14.336,2 +-14.224,2 +-14.114,2 +-14.008,2 +-13.901,2 +-13.789,2 +-13.677,2 +-13.564,2 +-13.458,2 +-13.345,2 +-13.242,2 +-13.129,2 +-13.018,2 +-12.909,2 +-12.8,2 +-12.688,2 +-12.575,2 +-12.473,2 +-12.361,2 +-12.252,2 +-12.139,2 +-12.027,2 +-11.917,2 +-11.808,2 +-11.701,2 +-11.592,2 +-11.482,2 +-11.374,2 +-11.268,2 +-11.16,2 +-11.054,2 +-10.943,2 +-10.838,2 +-10.731,2 +-10.627,2 +-10.514,2 +-10.408,2 +-10.298,2 +-10.184,2 +-10.075,2 +-9.969,2 +-9.861,2 +-9.756,2 +-9.644,2 +-9.532,2 +-9.425,2 +-9.314,2 +-9.201,2 +-9.088,2 +-8.984,2 +-8.871,2 +-8.759,2 +-8.654,2 +-8.541,2 +-8.428,2 +-8.317,2 +-8.208,2 +-8.095,2 +-7.982,2 +-7.875,2 +-7.768,2 +-7.659,2 +-7.547,2 +-7.436,2 +-7.333,2 +-7.222,2 +-7.112,2 +-7.007,2 +-6.895,2 +-6.782,2 +-6.67,2 +-6.559,2 +-6.452,2 +-6.345,2 +-6.234,2 +-6.125,2 +-6.017,2 +-5.909,2 +-5.799,2 +-5.687,2 +-5.575,2 +-5.465,2 +-5.354,2 +-5.248,2 +-5.142,2 +-5.04,2 +-4.934,2 +-4.829,2 +-4.717,2 +-4.609,2 +-4.5,2 +-4.392,2 +-4.284,2 +-4.18,2 +-4.068,2 +-3.96,2 +-3.853,2 +-3.743,2 +-3.63,2 +-3.519,2 +-3.413,2 +-3.301,2 +-3.198,2 +-3.087,2 +-2.975,2 +-2.864,2 +-2.758,2 +-2.649,2 +-2.544,2 +-2.434,2 +-2.326,2 +-2.214,2 +-2.1,2 +-1.986,2 +-1.875,2 +-1.771,2 +-1.659,2 +-1.55,2 +-1.437,2 +-1.326,2 +-1.217,2 +-1.108,2 +-0.999,2 +-0.892,2 +-0.779,2 +-0.666,2 +-0.555,2 +-0.445,2 +-0.335,2 +-0.227,2 +-0.117,2 +-0.011,2 +0.1,2 +0.206,2 +0.309,2 +0.416,2 +0.52,2 +0.632,2 +0.743,2 +0.85,2 +0.958,2 +1.069,2 +1.176,2 +1.288,2 +1.391,2 +1.498,1 +1.608,1 +1.718,1 +1.826,1 +1.933,1 +2.045,1 +2.151,1 +2.258,1 +2.368,1 +2.477,1 +2.591,1 +2.697,1 +2.8,1 +2.902,1 +3.009,1 +3.112,1 +3.215,1 +3.326,1 +3.433,1 +3.542,1 +3.652,1 +3.762,1 +3.873,1 +3.982,1 +4.09,1 +4.202,1 +4.315,1 +4.421,1 +4.533,1 +4.641,1 +4.746,1 +4.858,1 +4.966,1 +5.077,1 +5.182,1 +5.291,1 +5.405,1 +5.519,1 +5.628,1 +5.733,1 +5.839,1 +5.952,1 +6.057,1 +6.166,1 +6.271,1 +6.376,1 +6.479,1 +6.583,1 +6.691,1 +6.8,1 +6.912,1 +7.024,1 +7.136,1 +7.241,1 +7.351,1 +7.457,1 +7.566,1 +7.677,1 +7.79,1 +7.896,1 +8.008,1 +8.12,1 +8.224,1 +8.338,1 +8.451,1 +8.564,1 +8.677,1 +8.79,1 +8.894,1 +8.999,1 +9.109,1 +9.213,1 +9.323,1 +9.428,1 +9.541,1 +9.654,1 +9.766,1 +9.879,1 +9.982,1 +10.094,1 +10.202,1 +10.315,1 +10.427,1 +10.538,1 +10.64,1 +10.753,1 +10.858,1 +10.964,1 +11.074,1 +11.178,1 +11.287,1 +11.399,1 +11.507,1 +11.617,1 +11.723,1 +11.83,1 +11.942,1 +12.054,1 +12.166,1 +12.273,1 +12.382,1 +12.49,1 +12.598,1 +12.71,1 +12.818,1 +12.93,1 +13.038,1 +13.145,1 +13.257,1 +13.366,1 +13.477,1 +13.588,1 +13.693,1 +13.797,1 +13.908,1 +14.02,1 +14.127,1 +14.237,1 +14.34,1 +14.452,1 +14.562,1 +14.666,1 +14.769,1 +14.882,1 +14.99,2 +15.101,2 +15.212,2 +15.314,2 +15.426,2 +15.534,2 +15.638,2 +15.743,2 +15.849,2 +15.961,2 +16.072,2 +16.184,2 +16.291,2 +16.396,2 +16.507,2 +16.619,2 +16.731,2 +16.837,2 +16.951,2 +17.062,2 +17.165,2 +17.274,2 +17.385,2 +17.495,2 +17.604,2 +17.716,2 +17.828,2 +17.94,2 +18.052,2 +18.161,2 +18.268,2 +18.372,2 +18.482,2 +18.593,2 +18.699,2 +18.81,2 +18.923,2 +19.03,2 +19.14,2 +19.25,2 +19.362,2 +19.473,2 +19.585,2 +19.695,2 +19.798,2 +19.903,2 +20.014,2 +20.127,2 +20.238,2 +20.351,2 +20.457,2 +20.562,2 +20.668,2 +20.778,2 +20.888,2 +20.997,2 +21.106,2 +21.218,2 +21.322,2 +21.436,2 +21.541,2 +21.654,2 +21.768,2 +21.882,2 +21.985,2 +22.097,2 +22.21,2 +22.322,2 +22.433,2 +22.54,2 +22.653,2 +22.76,2 +22.867,2 +22.978,2 +23.09,2 +23.199,2 +23.309,2 +23.414,2 +23.521,2 +23.623,2 +23.734,2 +23.845,2 +23.949,2 +24.058,2 +24.165,2 +24.268,2 +24.37,2 +24.48,2 +24.582,2 +24.691,2 +24.802,2 +24.908,2 +25.022,2 +25.136,2 +25.25,2 +25.354,2 +25.466,2 +25.578,2 +25.691,2 +25.803,2 +25.907,2 +26.015,2 +26.125,2 +26.232,2 +26.335,2 +26.443,2 +26.555,2 +26.668,2 +26.773,2 +26.878,2 +26.99,2 +27.103,2 +27.217,2 +27.321,2 +27.433,2 +27.54,2 +27.651,2 +27.764,2 +27.876,2 +27.988,2 +28.099,2 +28.211,2 +28.322,2 +28.435,2 +28.537,2 +28.649,2 +28.76,2 +28.863,2 +28.972,2 +29.084,2 +29.196,2 +29.308,2 +29.417,2 +29.528,2 +29.637,2 +29.742,2 +29.848,2 +29.956,2 +30.069,2 +30.177,2 +30.29,2 +30.392,2 +30.498,2 +30.607,2 +30.714,2 +30.823,2 +30.926,2 +31.035,2 +31.139,2 +31.242,2 +31.351,2 +31.461,2 +31.567,2 +31.678,2 +31.79,2 +31.898,2 +32.012,2 +32.123,2 +32.231,2 +32.344,2 +32.457,2 +32.569,2 +32.681,2 +32.788,2 +32.898,2 +33.01,2 +33.122,2 +33.231,2 +33.346,2 +33.457,2 +33.565,2 +33.673,2 +33.783,2 +33.896,2 +34.006,2 +34.12,2 +34.223,2 +34.33,2 +34.443,2 +34.555,2 +34.665,2 +34.773,2 +34.881,2 +34.986,2 +35.096,2 +35.209,2 +35.313,2 +35.422,2 +35.534,2 +35.638,2 +35.75,2 +35.856,2 +35.965,2 +36.068,2 +36.181,2 +36.29,2 +36.398,2 +36.507,2 +36.615,2 +36.728,2 +36.84,2 +36.953,2 +37.065,2 +37.177,2 +37.289,2 +37.4,2 +37.504,2 +37.614,2 +37.722,2 +37.834,2 +37.94,2 +38.047,2 +38.154,2 +38.267,2 +38.371,2 +38.483,2 +38.593,2 +38.705,2 +38.814,2 +38.918,2 +39.023,2 +39.135,2 +39.237,2 +39.341,2 +39.446,2 +39.553,2 +39.656,2 +39.765,2 +39.873,2 +39.979,2 +40.081,2 +40.192,2 +40.299,2 +40.403,2 +40.507,2 +40.614,2 +40.726,2 +40.83,2 +40.941,2 +41.047,2 +41.158,2 +41.261,2 +41.37,2 +41.481,2 +41.589,2 +41.697,2 +41.808,2 +41.916,2 +42.018,2 +42.123,2 +42.236,2 +42.339,2 +42.448,2 +42.559,2 +42.664,2 +42.777,2 +42.889,2 +42.998,2 +43.105,2 +43.217,2 +43.328,2 +43.441,2 +43.554,2 +43.667,2 +43.774,2 +43.88,2 +43.99,2 +44.102,2 +44.206,2 +44.315,2 +44.426,2 +44.53,2 +44.636,2 +44.741,2 +44.846,2 +44.958,2 +45.068,2 +45.181,2 +45.285,2 +45.394,2 +45.497,2 diff --git a/evidence/failover-each/5bbf259ff136/probe.csv b/evidence/failover-each/5bbf259ff136/probe.csv new file mode 100644 index 00000000..176f9302 --- /dev/null +++ b/evidence/failover-each/5bbf259ff136/probe.csv @@ -0,0 +1,644 @@ +# date: 2026-11-05T19:18:13.470607+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +t_rel_s,status,latency_ms,instance +-25.748,201,3.4,7cfd91ddf780 +-25.644,201,5.5,5bbf259ff136 +-25.536,201,4.8,7cfd91ddf780 +-25.431,201,3.4,7cfd91ddf780 +-25.325,201,3.2,7cfd91ddf780 +-25.217,201,4.9,5bbf259ff136 +-25.107,201,3.4,7cfd91ddf780 +-24.997,201,3.0,7cfd91ddf780 +-24.884,201,3.5,7cfd91ddf780 +-24.778,201,5.8,7cfd91ddf780 +-24.664,201,5.3,7cfd91ddf780 +-24.558,201,6.9,7cfd91ddf780 +-24.442,201,4.8,5bbf259ff136 +-24.331,201,5.8,5bbf259ff136 +-24.215,201,5.9,5bbf259ff136 +-24.1,201,5.1,7cfd91ddf780 +-23.99,201,5.2,7cfd91ddf780 +-23.875,201,5.0,7cfd91ddf780 +-23.766,201,4.3,7cfd91ddf780 +-23.657,201,5.1,7cfd91ddf780 +-23.541,201,5.3,5bbf259ff136 +-23.426,201,6.7,5bbf259ff136 +-23.318,201,5.7,5bbf259ff136 +-23.208,201,5.1,5bbf259ff136 +-23.092,201,6.1,7cfd91ddf780 +-22.978,201,4.6,7cfd91ddf780 +-22.864,201,4.3,5bbf259ff136 +-22.759,201,4.6,7cfd91ddf780 +-22.644,201,5.3,5bbf259ff136 +-22.531,201,5.7,5bbf259ff136 +-22.417,201,4.9,5bbf259ff136 +-22.308,201,4.6,5bbf259ff136 +-22.197,201,5.7,5bbf259ff136 +-22.082,201,5.0,7cfd91ddf780 +-21.967,201,5.8,7cfd91ddf780 +-21.852,201,5.3,5bbf259ff136 +-21.74,201,4.5,5bbf259ff136 +-21.625,201,6.9,7cfd91ddf780 +-21.515,201,6.3,5bbf259ff136 +-21.402,201,5.0,7cfd91ddf780 +-21.287,201,4.5,7cfd91ddf780 +-21.179,201,4.6,7cfd91ddf780 +-21.067,201,5.3,7cfd91ddf780 +-20.95,201,6.0,5bbf259ff136 +-20.835,201,3.9,5bbf259ff136 +-20.724,201,3.8,7cfd91ddf780 +-20.617,201,4.8,5bbf259ff136 +-20.506,201,6.4,5bbf259ff136 +-20.392,201,5.9,7cfd91ddf780 +-20.284,201,5.1,7cfd91ddf780 +-20.175,201,7.0,5bbf259ff136 +-20.068,201,4.9,5bbf259ff136 +-19.958,201,5.2,5bbf259ff136 +-19.843,201,5.2,7cfd91ddf780 +-19.728,201,4.6,7cfd91ddf780 +-19.622,201,6.1,7cfd91ddf780 +-19.514,201,5.2,7cfd91ddf780 +-19.405,201,5.7,7cfd91ddf780 +-19.295,201,4.9,5bbf259ff136 +-19.182,201,4.7,7cfd91ddf780 +-19.076,201,6.0,5bbf259ff136 +-18.967,201,5.8,5bbf259ff136 +-18.86,201,5.6,5bbf259ff136 +-18.748,201,6.3,5bbf259ff136 +-18.642,201,6.0,5bbf259ff136 +-18.533,201,5.0,5bbf259ff136 +-18.426,201,5.5,7cfd91ddf780 +-18.315,201,5.3,5bbf259ff136 +-18.208,201,6.1,7cfd91ddf780 +-18.092,201,4.6,5bbf259ff136 +-17.979,201,5.3,5bbf259ff136 +-17.867,201,5.5,5bbf259ff136 +-17.752,201,3.8,7cfd91ddf780 +-17.643,201,5.2,7cfd91ddf780 +-17.531,201,5.1,7cfd91ddf780 +-17.415,201,7.0,5bbf259ff136 +-17.3,201,5.9,5bbf259ff136 +-17.187,201,5.5,5bbf259ff136 +-17.076,201,7.6,7cfd91ddf780 +-16.967,201,5.9,5bbf259ff136 +-16.86,201,4.9,5bbf259ff136 +-16.747,201,6.0,5bbf259ff136 +-16.631,201,6.8,7cfd91ddf780 +-16.516,201,5.4,7cfd91ddf780 +-16.407,201,4.9,7cfd91ddf780 +-16.297,201,4.4,7cfd91ddf780 +-16.188,201,4.9,7cfd91ddf780 +-16.073,201,5.0,5bbf259ff136 +-15.964,201,5.2,5bbf259ff136 +-15.849,201,4.9,5bbf259ff136 +-15.74,201,3.9,5bbf259ff136 +-15.636,201,3.4,7cfd91ddf780 +-15.523,201,5.0,5bbf259ff136 +-15.416,201,4.3,7cfd91ddf780 +-15.302,201,5.4,5bbf259ff136 +-15.192,201,5.1,5bbf259ff136 +-15.077,201,5.5,5bbf259ff136 +-14.969,201,6.1,7cfd91ddf780 +-14.859,201,5.1,7cfd91ddf780 +-14.745,201,5.3,7cfd91ddf780 +-14.63,201,6.8,5bbf259ff136 +-14.514,201,5.5,5bbf259ff136 +-14.398,201,6.2,7cfd91ddf780 +-14.282,201,6.5,5bbf259ff136 +-14.168,201,7.5,5bbf259ff136 +-14.059,201,6.1,5bbf259ff136 +-13.952,201,5.3,5bbf259ff136 +-13.838,201,5.4,5bbf259ff136 +-13.727,201,6.0,5bbf259ff136 +-13.611,201,4.8,7cfd91ddf780 +-13.504,201,6.4,5bbf259ff136 +-13.394,201,6.9,7cfd91ddf780 +-13.283,201,5.9,7cfd91ddf780 +-13.172,201,6.3,7cfd91ddf780 +-13.059,201,4.9,7cfd91ddf780 +-12.948,201,4.7,7cfd91ddf780 +-12.835,201,7.7,7cfd91ddf780 +-12.722,201,6.1,5bbf259ff136 +-12.607,201,5.3,5bbf259ff136 +-12.5,201,5.3,5bbf259ff136 +-12.395,201,6.2,7cfd91ddf780 +-12.288,201,4.9,7cfd91ddf780 +-12.18,201,5.3,7cfd91ddf780 +-12.074,201,6.5,7cfd91ddf780 +-11.967,201,5.8,5bbf259ff136 +-11.859,201,4.6,5bbf259ff136 +-11.747,201,5.1,5bbf259ff136 +-11.637,201,4.7,5bbf259ff136 +-11.525,201,5.4,5bbf259ff136 +-11.41,201,5.2,7cfd91ddf780 +-11.296,201,5.5,7cfd91ddf780 +-11.189,201,5.4,7cfd91ddf780 +-11.083,201,5.3,7cfd91ddf780 +-10.968,201,4.5,7cfd91ddf780 +-10.858,201,5.1,7cfd91ddf780 +-10.75,201,5.1,7cfd91ddf780 +-10.643,201,6.3,7cfd91ddf780 +-10.534,201,3.8,7cfd91ddf780 +-10.427,201,4.0,5bbf259ff136 +-10.32,201,5.9,5bbf259ff136 +-10.209,201,5.5,5bbf259ff136 +-10.095,201,5.4,5bbf259ff136 +-9.98,201,4.9,7cfd91ddf780 +-9.868,201,5.4,7cfd91ddf780 +-9.761,201,6.5,7cfd91ddf780 +-9.644,201,4.8,5bbf259ff136 +-9.531,201,6.3,5bbf259ff136 +-9.417,201,5.1,5bbf259ff136 +-9.303,201,4.8,7cfd91ddf780 +-9.192,201,5.6,7cfd91ddf780 +-9.084,201,4.8,7cfd91ddf780 +-8.975,201,4.7,5bbf259ff136 +-8.863,201,4.7,7cfd91ddf780 +-8.75,201,4.8,7cfd91ddf780 +-8.643,201,4.7,7cfd91ddf780 +-8.536,201,5.2,5bbf259ff136 +-8.424,201,5.2,5bbf259ff136 +-8.318,201,7.7,5bbf259ff136 +-8.205,201,5.1,5bbf259ff136 +-8.098,201,5.7,5bbf259ff136 +-7.983,201,6.8,7cfd91ddf780 +-7.869,201,4.6,7cfd91ddf780 +-7.76,201,5.0,7cfd91ddf780 +-7.645,201,5.5,7cfd91ddf780 +-7.531,201,5.0,5bbf259ff136 +-7.425,201,3.3,7cfd91ddf780 +-7.315,201,5.0,7cfd91ddf780 +-7.202,201,5.6,5bbf259ff136 +-7.092,201,5.2,5bbf259ff136 +-6.985,201,4.9,7cfd91ddf780 +-6.877,201,6.5,7cfd91ddf780 +-6.76,201,5.0,5bbf259ff136 +-6.65,201,4.6,5bbf259ff136 +-6.536,201,5.2,5bbf259ff136 +-6.423,201,5.4,7cfd91ddf780 +-6.307,201,4.0,7cfd91ddf780 +-6.203,201,4.3,7cfd91ddf780 +-6.093,201,4.3,7cfd91ddf780 +-5.985,201,7.2,5bbf259ff136 +-5.868,201,4.4,7cfd91ddf780 +-5.761,201,4.2,7cfd91ddf780 +-5.657,201,4.9,5bbf259ff136 +-5.546,201,5.7,5bbf259ff136 +-5.434,201,4.5,7cfd91ddf780 +-5.325,201,3.8,5bbf259ff136 +-5.216,201,3.9,5bbf259ff136 +-5.112,201,5.1,5bbf259ff136 +-5.0,201,6.2,5bbf259ff136 +-4.893,201,5.2,5bbf259ff136 +-4.784,201,6.3,5bbf259ff136 +-4.675,201,5.5,7cfd91ddf780 +-4.567,201,4.6,7cfd91ddf780 +-4.453,201,4.8,5bbf259ff136 +-4.338,201,5.0,5bbf259ff136 +-4.229,201,6.4,5bbf259ff136 +-4.115,201,6.0,5bbf259ff136 +-4.007,201,5.4,5bbf259ff136 +-3.896,201,6.5,5bbf259ff136 +-3.782,201,5.5,7cfd91ddf780 +-3.669,201,5.5,7cfd91ddf780 +-3.562,201,5.4,7cfd91ddf780 +-3.452,201,5.5,7cfd91ddf780 +-3.341,201,5.9,7cfd91ddf780 +-3.234,201,6.5,7cfd91ddf780 +-3.117,201,5.1,7cfd91ddf780 +-3.007,201,6.6,7cfd91ddf780 +-2.892,201,6.3,5bbf259ff136 +-2.775,201,5.6,5bbf259ff136 +-2.669,201,6.0,5bbf259ff136 +-2.559,201,6.6,5bbf259ff136 +-2.442,201,6.4,7cfd91ddf780 +-2.326,201,3.6,7cfd91ddf780 +-2.214,201,5.1,5bbf259ff136 +-2.099,201,4.8,5bbf259ff136 +-1.986,201,6.5,5bbf259ff136 +-1.877,201,7.0,5bbf259ff136 +-1.768,201,4.7,5bbf259ff136 +-1.658,201,5.8,5bbf259ff136 +-1.548,201,5.2,5bbf259ff136 +-1.433,201,5.4,7cfd91ddf780 +-1.326,201,7.0,5bbf259ff136 +-1.217,201,5.3,5bbf259ff136 +-1.108,201,5.1,5bbf259ff136 +-0.995,201,4.4,5bbf259ff136 +-0.884,201,5.8,7cfd91ddf780 +-0.776,201,5.7,5bbf259ff136 +-0.662,201,5.5,7cfd91ddf780 +-0.554,201,6.5,7cfd91ddf780 +-0.442,201,4.7,7cfd91ddf780 +-0.331,201,5.1,7cfd91ddf780 +-0.223,201,3.6,5bbf259ff136 +-0.111,201,3.9,7cfd91ddf780 +-0.004,201,5.0,5bbf259ff136 +0.106,201,4.8,7cfd91ddf780 +0.217,201,5.2,5bbf259ff136 +0.323,201,6.3,5bbf259ff136 +0.44,201,5.2,7cfd91ddf780 +0.547,201,5.5,7cfd91ddf780 +0.657,201,3.2,7cfd91ddf780 +0.771,201,2.8,7cfd91ddf780 +0.879,201,3.9,7cfd91ddf780 +0.993,201,3.9,7cfd91ddf780 +1.107,201,4.7,5bbf259ff136 +1.222,201,4.9,5bbf259ff136 +1.333,201,3.9,5bbf259ff136 +1.439,201,6.0,5bbf259ff136 +1.551,201,6.0,7cfd91ddf780 +1.66,201,6.1,7cfd91ddf780 +1.775,201,5.1,7cfd91ddf780 +1.887,201,7.0,7cfd91ddf780 +2.004,201,5.9,7cfd91ddf780 +2.12,201,5.4,7cfd91ddf780 +2.233,201,4.1,7cfd91ddf780 +2.338,201,3.8,7cfd91ddf780 +2.449,201,6.8,7cfd91ddf780 +2.557,201,4.3,7cfd91ddf780 +2.661,201,6.9,7cfd91ddf780 +2.768,201,4.5,7cfd91ddf780 +2.876,201,4.2,7cfd91ddf780 +2.983,201,4.6,7cfd91ddf780 +3.098,201,4.4,7cfd91ddf780 +3.203,201,5.2,7cfd91ddf780 +3.31,201,5.2,7cfd91ddf780 +3.425,201,5.5,7cfd91ddf780 +3.537,201,5.9,7cfd91ddf780 +3.646,201,8.7,7cfd91ddf780 +3.758,201,5.7,7cfd91ddf780 +3.867,201,7.2,7cfd91ddf780 +3.982,201,6.7,7cfd91ddf780 +4.099,201,5.7,7cfd91ddf780 +4.206,201,4.6,7cfd91ddf780 +4.316,201,13.6,7cfd91ddf780 +4.439,201,4.2,7cfd91ddf780 +4.554,201,5.9,7cfd91ddf780 +4.665,201,7.6,7cfd91ddf780 +4.781,201,4.8,7cfd91ddf780 +4.896,201,5.9,7cfd91ddf780 +5.008,201,5.5,7cfd91ddf780 +5.123,201,4.3,7cfd91ddf780 +5.233,201,4.4,7cfd91ddf780 +5.338,201,5.1,7cfd91ddf780 +5.449,201,5.8,7cfd91ddf780 +5.557,201,5.6,7cfd91ddf780 +5.673,201,6.9,7cfd91ddf780 +5.781,201,6.3,7cfd91ddf780 +5.887,201,6.8,7cfd91ddf780 +6.004,201,5.0,7cfd91ddf780 +6.118,201,5.1,7cfd91ddf780 +6.232,201,5.5,7cfd91ddf780 +6.347,201,5.2,7cfd91ddf780 +6.454,201,5.8,7cfd91ddf780 +6.564,201,5.1,7cfd91ddf780 +6.674,201,4.4,7cfd91ddf780 +6.783,201,5.7,7cfd91ddf780 +6.892,201,3.0,7cfd91ddf780 +7.004,201,6.7,7cfd91ddf780 +7.119,201,6.2,7cfd91ddf780 +7.235,201,5.3,7cfd91ddf780 +7.351,201,6.6,7cfd91ddf780 +7.458,201,6.4,7cfd91ddf780 +7.568,201,5.2,7cfd91ddf780 +7.677,201,7.3,7cfd91ddf780 +7.793,201,5.2,7cfd91ddf780 +7.899,201,3.3,7cfd91ddf780 +8.012,201,2.8,7cfd91ddf780 +8.124,201,3.9,7cfd91ddf780 +8.232,201,4.9,7cfd91ddf780 +8.337,201,6.0,7cfd91ddf780 +8.449,201,5.9,7cfd91ddf780 +8.563,201,5.8,7cfd91ddf780 +8.674,201,7.4,7cfd91ddf780 +8.782,201,4.3,7cfd91ddf780 +8.894,201,6.0,7cfd91ddf780 +9.01,201,4.9,7cfd91ddf780 +9.122,201,5.6,7cfd91ddf780 +9.233,201,6.0,7cfd91ddf780 +9.339,201,5.2,7cfd91ddf780 +9.449,201,6.0,7cfd91ddf780 +9.556,201,6.4,7cfd91ddf780 +9.673,201,5.0,7cfd91ddf780 +9.788,201,5.0,7cfd91ddf780 +9.899,201,4.8,7cfd91ddf780 +10.009,201,5.2,7cfd91ddf780 +10.12,201,5.8,7cfd91ddf780 +10.233,201,4.7,7cfd91ddf780 +10.348,201,4.6,7cfd91ddf780 +10.453,201,4.3,7cfd91ddf780 +10.566,201,5.3,7cfd91ddf780 +10.681,201,5.9,7cfd91ddf780 +10.789,201,7.5,7cfd91ddf780 +10.899,201,6.3,7cfd91ddf780 +11.012,201,6.7,7cfd91ddf780 +11.122,201,4.8,7cfd91ddf780 +11.228,201,5.5,7cfd91ddf780 +11.343,201,4.9,7cfd91ddf780 +11.458,201,5.8,7cfd91ddf780 +11.573,201,5.4,7cfd91ddf780 +11.688,201,6.7,7cfd91ddf780 +11.805,201,5.5,7cfd91ddf780 +11.916,201,6.0,7cfd91ddf780 +12.023,201,5.8,7cfd91ddf780 +12.137,201,6.0,7cfd91ddf780 +12.244,201,4.7,7cfd91ddf780 +12.355,201,6.5,7cfd91ddf780 +12.466,201,5.1,7cfd91ddf780 +12.578,201,5.1,7cfd91ddf780 +12.69,201,5.4,7cfd91ddf780 +12.8,201,5.2,7cfd91ddf780 +12.915,201,4.0,7cfd91ddf780 +13.026,201,4.9,7cfd91ddf780 +13.138,201,3.7,7cfd91ddf780 +13.242,201,4.3,7cfd91ddf780 +13.351,201,3.3,7cfd91ddf780 +13.455,201,3.9,7cfd91ddf780 +13.569,201,3.5,7cfd91ddf780 +13.676,201,3.9,7cfd91ddf780 +13.782,201,3.1,7cfd91ddf780 +13.89,201,3.8,7cfd91ddf780 +13.999,201,4.0,7cfd91ddf780 +14.103,201,4.8,7cfd91ddf780 +14.21,201,4.8,7cfd91ddf780 +14.318,201,5.0,7cfd91ddf780 +14.43,201,5.3,7cfd91ddf780 +14.543,201,5.1,7cfd91ddf780 +14.65,201,5.2,7cfd91ddf780 +14.756,201,4.8,7cfd91ddf780 +14.863,201,5.7,7cfd91ddf780 +14.979,201,5.9,7cfd91ddf780 +15.093,201,5.5,7cfd91ddf780 +15.203,201,5.1,7cfd91ddf780 +15.309,201,4.9,7cfd91ddf780 +15.416,201,8.0,7cfd91ddf780 +15.534,201,6.0,7cfd91ddf780 +15.642,201,3.5,7cfd91ddf780 +15.756,201,4.7,5bbf259ff136 +15.871,201,6.0,5bbf259ff136 +15.987,201,4.7,7cfd91ddf780 +16.097,201,4.9,7cfd91ddf780 +16.212,201,5.2,7cfd91ddf780 +16.324,201,6.2,7cfd91ddf780 +16.44,201,5.5,5bbf259ff136 +16.556,201,4.9,7cfd91ddf780 +16.666,201,5.2,7cfd91ddf780 +16.777,201,6.3,7cfd91ddf780 +16.893,201,5.8,7cfd91ddf780 +17.009,201,5.4,5bbf259ff136 +17.123,201,5.5,5bbf259ff136 +17.229,201,5.4,5bbf259ff136 +17.343,201,5.1,5bbf259ff136 +17.457,201,5.4,7cfd91ddf780 +17.564,201,5.2,5bbf259ff136 +17.679,201,7.2,7cfd91ddf780 +17.79,201,6.8,7cfd91ddf780 +17.899,201,5.0,7cfd91ddf780 +18.013,201,5.1,7cfd91ddf780 +18.126,201,8.5,5bbf259ff136 +18.241,201,3.9,5bbf259ff136 +18.352,201,4.5,5bbf259ff136 +18.465,201,5.5,7cfd91ddf780 +18.577,201,5.9,7cfd91ddf780 +18.691,201,4.8,7cfd91ddf780 +18.806,201,14.2,5bbf259ff136 +18.92,201,5.3,5bbf259ff136 +19.03,201,4.6,5bbf259ff136 +19.141,201,5.5,5bbf259ff136 +19.249,201,5.7,5bbf259ff136 +19.357,201,4.8,5bbf259ff136 +19.468,201,5.1,5bbf259ff136 +19.582,201,6.6,5bbf259ff136 +19.695,201,5.3,7cfd91ddf780 +19.802,201,5.1,5bbf259ff136 +19.912,201,4.9,5bbf259ff136 +20.021,201,5.2,5bbf259ff136 +20.136,201,5.4,7cfd91ddf780 +20.251,201,4.2,7cfd91ddf780 +20.361,201,5.0,7cfd91ddf780 +20.473,201,5.2,7cfd91ddf780 +20.582,201,5.0,7cfd91ddf780 +20.692,201,5.9,7cfd91ddf780 +20.805,201,4.7,7cfd91ddf780 +20.912,201,4.5,7cfd91ddf780 +21.023,201,4.7,7cfd91ddf780 +21.137,201,5.2,5bbf259ff136 +21.249,201,4.4,5bbf259ff136 +21.362,201,5.1,5bbf259ff136 +21.472,201,5.0,5bbf259ff136 +21.583,201,5.6,5bbf259ff136 +21.691,201,6.1,5bbf259ff136 +21.8,201,7.2,5bbf259ff136 +21.917,201,6.8,7cfd91ddf780 +22.03,201,5.0,7cfd91ddf780 +22.137,201,6.5,7cfd91ddf780 +22.253,201,7.0,7cfd91ddf780 +22.364,201,5.3,7cfd91ddf780 +22.474,201,5.3,7cfd91ddf780 +22.582,201,5.7,7cfd91ddf780 +22.691,201,5.6,7cfd91ddf780 +22.801,201,4.9,7cfd91ddf780 +22.916,201,4.4,5bbf259ff136 +23.02,201,4.9,7cfd91ddf780 +23.135,201,5.3,5bbf259ff136 +23.242,201,6.3,7cfd91ddf780 +23.358,201,4.7,5bbf259ff136 +23.467,201,4.5,5bbf259ff136 +23.572,201,4.1,7cfd91ddf780 +23.682,201,5.6,7cfd91ddf780 +23.79,201,4.8,7cfd91ddf780 +23.905,201,5.1,5bbf259ff136 +24.015,201,5.0,5bbf259ff136 +24.122,201,4.7,7cfd91ddf780 +24.233,201,4.4,7cfd91ddf780 +24.342,201,5.8,7cfd91ddf780 +24.458,201,6.0,5bbf259ff136 +24.569,201,5.4,5bbf259ff136 +24.684,201,6.7,7cfd91ddf780 +24.801,201,5.3,7cfd91ddf780 +24.916,201,5.2,5bbf259ff136 +25.023,201,6.6,7cfd91ddf780 +25.14,201,5.7,5bbf259ff136 +25.251,201,4.5,5bbf259ff136 +25.357,201,5.0,5bbf259ff136 +25.472,201,5.1,5bbf259ff136 +25.582,201,4.8,5bbf259ff136 +25.691,201,6.0,5bbf259ff136 +25.807,201,5.2,7cfd91ddf780 +25.915,201,4.9,7cfd91ddf780 +26.03,201,5.1,7cfd91ddf780 +26.14,201,5.1,7cfd91ddf780 +26.247,201,5.6,7cfd91ddf780 +26.354,201,5.3,5bbf259ff136 +26.467,201,6.8,7cfd91ddf780 +26.583,201,5.2,7cfd91ddf780 +26.694,201,5.2,7cfd91ddf780 +26.806,201,6.2,5bbf259ff136 +26.914,201,6.5,5bbf259ff136 +27.031,201,5.9,5bbf259ff136 +27.144,201,5.4,7cfd91ddf780 +27.255,201,5.1,7cfd91ddf780 +27.362,201,6.0,5bbf259ff136 +27.478,201,6.3,7cfd91ddf780 +27.59,201,6.3,7cfd91ddf780 +27.707,201,5.4,5bbf259ff136 +27.814,201,7.4,5bbf259ff136 +27.929,201,3.9,5bbf259ff136 +28.04,201,4.0,5bbf259ff136 +28.153,201,5.2,5bbf259ff136 +28.269,201,4.8,7cfd91ddf780 +28.381,201,5.6,7cfd91ddf780 +28.487,201,5.1,7cfd91ddf780 +28.597,201,5.1,7cfd91ddf780 +28.708,201,3.9,7cfd91ddf780 +28.816,201,5.1,7cfd91ddf780 +28.931,201,4.9,7cfd91ddf780 +29.046,201,5.9,5bbf259ff136 +29.162,201,4.9,5bbf259ff136 +29.276,201,3.8,7cfd91ddf780 +29.381,201,5.3,5bbf259ff136 +29.489,201,4.7,5bbf259ff136 +29.6,201,4.9,5bbf259ff136 +29.715,201,5.9,7cfd91ddf780 +29.825,201,5.5,7cfd91ddf780 +29.94,201,5.9,7cfd91ddf780 +30.047,201,7.0,7cfd91ddf780 +30.155,201,5.5,7cfd91ddf780 +30.271,201,6.1,7cfd91ddf780 +30.38,201,4.9,7cfd91ddf780 +30.49,201,5.3,7cfd91ddf780 +30.597,201,7.2,7cfd91ddf780 +30.714,201,6.6,5bbf259ff136 +30.822,201,6.9,7cfd91ddf780 +30.933,201,6.4,7cfd91ddf780 +31.049,201,3.8,5bbf259ff136 +31.163,201,4.9,5bbf259ff136 +31.273,201,5.4,5bbf259ff136 +31.383,201,5.1,5bbf259ff136 +31.49,201,5.0,5bbf259ff136 +31.605,201,5.9,7cfd91ddf780 +31.717,201,6.5,7cfd91ddf780 +31.825,201,5.7,7cfd91ddf780 +31.938,201,5.0,7cfd91ddf780 +32.053,201,4.8,7cfd91ddf780 +32.164,201,4.9,7cfd91ddf780 +32.271,201,4.6,7cfd91ddf780 +32.378,201,4.7,7cfd91ddf780 +32.49,201,5.0,7cfd91ddf780 +32.605,201,4.2,7cfd91ddf780 +32.717,201,4.1,5bbf259ff136 +32.828,201,5.4,5bbf259ff136 +32.937,201,6.0,5bbf259ff136 +33.048,201,6.2,5bbf259ff136 +33.163,201,5.3,5bbf259ff136 +33.276,201,6.7,7cfd91ddf780 +33.39,201,6.6,7cfd91ddf780 +33.501,201,5.0,7cfd91ddf780 +33.616,201,5.3,5bbf259ff136 +33.722,201,4.9,7cfd91ddf780 +33.831,201,3.9,7cfd91ddf780 +33.945,201,5.9,5bbf259ff136 +34.06,201,5.4,5bbf259ff136 +34.168,201,5.5,5bbf259ff136 +34.28,201,5.6,5bbf259ff136 +34.392,201,3.1,5bbf259ff136 +34.505,201,5.5,7cfd91ddf780 +34.621,201,4.7,7cfd91ddf780 +34.736,201,5.2,5bbf259ff136 +34.851,201,6.5,5bbf259ff136 +34.965,201,5.7,7cfd91ddf780 +35.072,201,4.9,5bbf259ff136 +35.182,201,6.6,5bbf259ff136 +35.291,201,4.8,5bbf259ff136 +35.403,201,6.7,5bbf259ff136 +35.518,201,6.8,7cfd91ddf780 +35.627,201,4.7,7cfd91ddf780 +35.732,201,6.7,5bbf259ff136 +35.848,201,5.8,7cfd91ddf780 +35.962,201,6.7,7cfd91ddf780 +36.073,201,5.2,7cfd91ddf780 +36.182,201,6.8,7cfd91ddf780 +36.29,201,6.2,7cfd91ddf780 +36.405,201,5.0,5bbf259ff136 +36.52,201,5.0,5bbf259ff136 +36.634,201,5.0,5bbf259ff136 +36.747,201,4.6,7cfd91ddf780 +36.861,201,5.0,7cfd91ddf780 +36.974,201,5.6,5bbf259ff136 +37.088,201,4.8,5bbf259ff136 +37.203,201,5.6,5bbf259ff136 +37.314,201,5.5,7cfd91ddf780 +37.43,201,5.7,7cfd91ddf780 +37.545,201,4.8,7cfd91ddf780 +37.659,201,4.3,5bbf259ff136 +37.765,201,5.4,5bbf259ff136 +37.873,201,5.3,7cfd91ddf780 +37.981,201,6.1,7cfd91ddf780 +38.097,201,6.0,5bbf259ff136 +38.214,201,6.3,7cfd91ddf780 +38.327,201,5.5,7cfd91ddf780 +38.44,201,6.0,7cfd91ddf780 +38.548,201,5.4,7cfd91ddf780 +38.656,201,6.9,7cfd91ddf780 +38.765,201,3.5,5bbf259ff136 +38.868,201,3.5,5bbf259ff136 +38.981,201,3.7,5bbf259ff136 +39.088,201,3.1,5bbf259ff136 +39.2,201,3.6,5bbf259ff136 +39.308,201,4.0,5bbf259ff136 +39.414,201,4.3,7cfd91ddf780 +39.523,201,3.5,7cfd91ddf780 +39.635,201,4.3,5bbf259ff136 +39.744,201,4.2,5bbf259ff136 +39.859,201,4.0,5bbf259ff136 +39.965,201,3.4,5bbf259ff136 +40.076,201,7.7,5bbf259ff136 +40.192,201,6.5,5bbf259ff136 +40.306,201,5.0,7cfd91ddf780 +40.414,201,3.5,5bbf259ff136 +40.518,201,2.6,5bbf259ff136 +40.623,201,2.6,7cfd91ddf780 +40.733,201,4.6,7cfd91ddf780 +40.845,201,4.6,5bbf259ff136 +40.958,201,4.2,5bbf259ff136 +41.064,201,4.2,7cfd91ddf780 +41.171,201,2.8,7cfd91ddf780 +41.281,201,4.9,7cfd91ddf780 +41.388,201,4.3,7cfd91ddf780 +41.497,201,5.0,7cfd91ddf780 +41.606,201,5.1,7cfd91ddf780 +41.721,201,2.9,7cfd91ddf780 +41.829,201,4.8,7cfd91ddf780 +41.942,201,3.0,7cfd91ddf780 +42.053,201,5.6,7cfd91ddf780 +42.164,201,6.5,7cfd91ddf780 +42.281,201,5.6,5bbf259ff136 +42.389,201,4.5,5bbf259ff136 +42.503,201,4.6,5bbf259ff136 +42.618,201,5.8,7cfd91ddf780 +42.734,201,5.2,7cfd91ddf780 +42.849,201,5.9,5bbf259ff136 +42.959,201,5.0,5bbf259ff136 +43.066,201,4.9,5bbf259ff136 +43.181,201,6.2,5bbf259ff136 +43.289,201,5.7,5bbf259ff136 +43.4,201,6.0,5bbf259ff136 +43.516,201,7.1,7cfd91ddf780 +43.631,201,6.1,7cfd91ddf780 +43.746,201,6.7,5bbf259ff136 +43.862,201,5.9,5bbf259ff136 +43.969,201,5.5,5bbf259ff136 +44.084,201,5.4,7cfd91ddf780 +44.2,201,4.6,7cfd91ddf780 +44.305,201,5.8,7cfd91ddf780 +44.421,201,6.1,7cfd91ddf780 +44.53,201,5.2,7cfd91ddf780 +44.637,201,3.7,7cfd91ddf780 +44.75,201,3.3,7cfd91ddf780 +44.855,201,3.8,5bbf259ff136 +44.961,201,4.7,5bbf259ff136 +45.072,201,6.1,7cfd91ddf780 +45.184,201,7.3,5bbf259ff136 +45.296,201,7.5,7cfd91ddf780 +45.409,201,5.2,5bbf259ff136 +45.521,201,5.5,7cfd91ddf780 diff --git a/evidence/failover-each/fe75c6c37e5a/result.json b/evidence/failover-each/5bbf259ff136/result.json similarity index 59% rename from evidence/failover-each/fe75c6c37e5a/result.json rename to evidence/failover-each/5bbf259ff136/result.json index 0b2b0034..c3c2e4f5 100644 --- a/evidence/failover-each/fe75c6c37e5a/result.json +++ b/evidence/failover-each/5bbf259ff136/result.json @@ -1,34 +1,42 @@ -{ - "label": "fe75c6c37e5a", +{"date": "2026-11-05T19:18:13.470607+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", + "label": "5bbf259ff136", "action": [ "sh", "-c", "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' api && { scripts/stand-usage.sh & sleep 10; wait; } && scripts/restore.sh" ], "action_exit": 0, - "action_seconds": 14.56, + "action_seconds": 15.2, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 1, - "detection_seconds": 0.859, - "rejoin_seconds": 14.343, + "lb_samples_missing": 0, + "detection_seconds": 1.498, + "rejoin_seconds": 14.99, "probe": { - "attempts": 646, + "attempts": 642, "failed_attempts": 0, "first_failure_s": null, + "first_success_s": 0.111, "recovery_s": 0.0, "instances_before": { - "fe75c6c37e5a": 130, - "ff81f93e8b0d": 105 + "7cfd91ddf780": 114, + "5bbf259ff136": 119 }, "instances_during_outage": { - "ff81f93e8b0d": 121 + "7cfd91ddf780": 121 }, "instances_after": { - "ff81f93e8b0d": 123, - "fe75c6c37e5a": 156 + "7cfd91ddf780": 149, + "5bbf259ff136": 124 } }, + "history": { + "confirmed": 642, + "stored": 642, + "exact_match": true, + "unique": true + }, "lb_counters": { "wretr": 0, "wredis": 0, @@ -42,16 +50,19 @@ "ok_rps": 100.02, "failed_ratio": 0, "dropped_iterations": 0, - "p50_ms": 1.57, - "p95_ms": 3.62, - "p99_ms": 6.24, + "p50_ms": 2.98, + "p95_ms": 3.88, + "p99_ms": 4.53, "exit": 0, "rate": 100 }, "checks": { "action_ok": true, "load_errors_ok": true, + "load_delivered": true, + "lb_observed": true, "write_recovery_ok": true, + "history_preserved": true, "detected": true, "target_excluded": true, "rejoined": true diff --git a/evidence/failover-each/7cfd91ddf780/action.txt b/evidence/failover-each/7cfd91ddf780/action.txt new file mode 100644 index 00000000..00dd91f6 --- /dev/null +++ b/evidence/failover-each/7cfd91ddf780/action.txt @@ -0,0 +1,15 @@ +date: 2026-11-05T19:18:13.470607+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +7cfd91ddf780 api Exited (0) Less than a second ago +5bbf259ff136 api Up 3 minutes (healthy) +19:18:43 database bytes,messages: 9385651,2542 +mtusi2026lab3-grade4-20261105-api-2 cpu=23.33% mem=217.6MiB / 512MiB +mtusi2026lab3-grade4-20261105-lb-1 cpu=3.14% mem=30.17MiB / 128MiB +mtusi2026lab3-grade4-20261105-postgres-1 cpu=6.03% mem=58.73MiB / 1GiB +mtusi2026lab3-grade4-20261105-valkey-1 cpu=0.42% mem=12.97MiB / 128MiB + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-1 Starting + Container mtusi2026lab3-grade4-20261105-api-1 Started + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy diff --git a/evidence/failover-each/7cfd91ddf780/k6/k6.txt b/evidence/failover-each/7cfd91ddf780/k6/k6.txt new file mode 100644 index 00000000..c6990295 --- /dev/null +++ b/evidence/failover-each/7cfd91ddf780/k6/k6.txt @@ -0,0 +1,3 @@ +date: 2026-11-05T19:18:13.470607+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +time="2026-11-05T19:18:15Z" level=info msg=LOAD-START source=console +measure: 6001 requests, failed ratio 0 diff --git a/evidence/failover-each/7cfd91ddf780/k6/summary.json b/evidence/failover-each/7cfd91ddf780/k6/summary.json new file mode 100644 index 00000000..f1259dc0 --- /dev/null +++ b/evidence/failover-each/7cfd91ddf780/k6/summary.json @@ -0,0 +1,309 @@ +{"date": "2026-11-05T19:18:13.470607+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", + "root_group": { + "checks": [ + { + "name": "message 201", + "path": "::message 201", + "id": "6445aff337e1ade0bc10402fbac79882", + "passes": 5614, + "fails": 0 + }, + { + "name": "history 200", + "path": "::history 200", + "id": "fe27b3b8edd292c7f2170a6357b087d0", + "passes": 1388, + "fails": 0 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "p(90)", + "p(95)", + "p(99)", + "max" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": false, + "isStdErrTTY": false, + "testRunDurationMs": 70538.919963 + }, + "metrics": { + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "p(90)": 0, + "p(95)": 0, + "p(99)": 0, + "max": 0, + "avg": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 7052 + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 3.943366299999998, + "p(99)": 7.941836829999957, + "max": 52.49775, + "avg": 3.1087168380601247, + "min": 1.647417, + "med": 2.9766874999999997, + "p(90)": 3.6688584 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 7002, + "fails": 0 + } + }, + "iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 6001, + "rate": 85.07360196537917 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 3.1087168380601247, + "min": 1.647417, + "med": 2.9766874999999997, + "p(90)": 3.6688584, + "p(95)": 3.943366299999998, + "p(99)": 7.941836829999957, + "max": 52.49775 + } + }, + "http_req_sending": { + "contains": "time", + "values": { + "max": 0.354375, + "avg": 0.01261951956891675, + "min": 0.00275, + "med": 0.011459, + "p(90)": 0.015125, + "p(95)": 0.01775, + "p(99)": 0.033123339999999994 + }, + "type": "trend" + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.022591304452637576, + "min": 0.008958, + "med": 0.019917, + "p(90)": 0.032, + "p(95)": 0.03826844999999999, + "p(99)": 0.06532848999999998, + "max": 0.210875 + } + }, + "data_sent": { + "values": { + "count": 2053317, + "rate": 29108.994028786277 + }, + "type": "counter", + "contains": "data" + }, + "data_received": { + "values": { + "count": 17042097, + "rate": 241598.4963894988 + }, + "type": "counter", + "contains": "data" + }, + "http_req_duration{scenario:measure}": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 3.862542, + "p(99)": 4.473916, + "max": 52.49775, + "avg": 3.049362646058984, + "min": 1.684875, + "med": 2.9745, + "p(90)": 3.631833 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "max": 0.224708, + "avg": 0.00034415215541690295, + "min": 0, + "med": 0, + "p(90)": 0, + "p(95)": 0, + "p(99)": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 1, + "min": 0, + "max": 1 + } + }, + "http_req_failed{scenario:measure}": { + "values": { + "fails": 6001, + "rate": 0, + "passes": 0 + }, + "type": "rate", + "contains": "default" + }, + "dropped_iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 0, + "rate": 0 + } + }, + "http_req_duration{scenario:measure,name:post_message}": { + "type": "trend", + "contains": "time", + "values": { + "med": 2.940125, + "p(90)": 3.551959, + "p(95)": 3.786104, + "p(99)": 4.346612499999998, + "max": 52.49775, + "avg": 3.0002135637081597, + "min": 1.684875 + } + }, + "http_req_waiting": { + "values": { + "max": 52.465792, + "avg": 3.073506014038576, + "min": 1.628417, + "med": 2.941875, + "p(90)": 3.6291869, + "p(95)": 3.8983437499999996, + "p(99)": 7.918076749999957 + }, + "type": "trend", + "contains": "time" + }, + "http_reqs{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 6001, + "rate": 85.07360196537917 + } + }, + "dropped_iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 0, + "rate": 0 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 7002, + "rate": 99.26434943535826 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "min": 0.000708, + "med": 0.001667, + "p(90)": 0.0025, + "p(95)": 0.002959, + "p(99)": 0.006083, + "max": 1.817417, + "avg": 0.0025302468803176624 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "min": 20, + "max": 20, + "value": 20 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 7052, + "rate": 99.97317798031226 + } + }, + "iteration_duration": { + "values": { + "avg": 3.1430217764924397, + "min": 1.686459, + "med": 3.0623125, + "p(90)": 3.7471503, + "p(95)": 4.001064599999999, + "p(99)": 4.828713, + "max": 52.592501 + }, + "type": "trend", + "contains": "time" + }, + "http_req_duration{scenario:measure,name:get_history}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 3.2480653647058872, + "min": 1.856375, + "med": 3.166271, + "p(90)": 3.8746452, + "p(95)": 4.109600299999999, + "p(99)": 5.1775943700000004, + "max": 17.655583 + } + } + } +} diff --git a/evidence/failover-each/7cfd91ddf780/lb_up.csv b/evidence/failover-each/7cfd91ddf780/lb_up.csv new file mode 100644 index 00000000..387a2d84 --- /dev/null +++ b/evidence/failover-each/7cfd91ddf780/lb_up.csv @@ -0,0 +1,663 @@ +# date: 2026-11-05T19:18:13.470607+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +t_rel_s,up +-25.852,2 +-25.745,2 +-25.643,2 +-25.532,2 +-25.426,2 +-25.319,2 +-25.21,2 +-25.1,2 +-24.995,2 +-24.894,2 +-24.79,2 +-24.686,2 +-24.577,2 +-24.469,2 +-24.358,2 +-24.255,2 +-24.148,2 +-24.042,2 +-23.939,2 +-23.836,2 +-23.727,2 +-23.625,2 +-23.521,2 +-23.412,2 +-23.303,2 +-23.192,2 +-23.088,2 +-22.978,2 +-22.868,2 +-22.757,2 +-22.648,2 +-22.541,2 +-22.44,2 +-22.33,2 +-22.227,2 +-22.121,2 +-22.019,2 +-21.908,2 +-21.804,2 +-21.699,2 +-21.588,2 +-21.476,2 +-21.363,2 +-21.259,2 +-21.148,2 +-21.036,2 +-20.928,2 +-20.82,2 +-20.709,2 +-20.604,2 +-20.493,2 +-20.381,2 +-20.269,2 +-20.158,2 +-20.053,2 +-19.944,2 +-19.837,2 +-19.729,2 +-19.619,2 +-19.511,2 +-19.406,2 +-19.304,2 +-19.191,2 +-19.083,2 +-18.972,2 +-18.869,2 +-18.761,2 +-18.649,2 +-18.542,2 +-18.43,2 +-18.318,2 +-18.216,2 +-18.114,2 +-18.005,2 +-17.899,2 +-17.792,2 +-17.685,2 +-17.581,2 +-17.472,2 +-17.359,2 +-17.253,2 +-17.148,2 +-17.043,2 +-16.936,2 +-16.823,2 +-16.712,2 +-16.602,2 +-16.495,2 +-16.386,2 +-16.276,2 +-16.164,2 +-16.061,2 +-15.957,2 +-15.846,2 +-15.736,2 +-15.632,2 +-15.526,2 +-15.422,2 +-15.311,2 +-15.205,2 +-15.101,2 +-14.991,2 +-14.879,2 +-14.768,2 +-14.661,2 +-14.554,2 +-14.444,2 +-14.335,2 +-14.228,2 +-14.123,2 +-14.018,2 +-13.906,2 +-13.792,2 +-13.68,2 +-13.574,2 +-13.465,2 +-13.358,2 +-13.247,2 +-13.141,2 +-13.029,2 +-12.922,2 +-12.809,2 +-12.697,2 +-12.586,2 +-12.474,2 +-12.37,2 +-12.26,2 +-12.154,2 +-12.041,2 +-11.935,2 +-11.824,2 +-11.719,2 +-11.608,2 +-11.506,2 +-11.397,2 +-11.283,2 +-11.176,2 +-11.07,2 +-10.962,2 +-10.851,2 +-10.741,2 +-10.629,2 +-10.519,2 +-10.408,2 +-10.296,2 +-10.185,2 +-10.078,2 +-9.975,2 +-9.87,2 +-9.76,2 +-9.653,2 +-9.54,2 +-9.43,2 +-9.319,2 +-9.211,2 +-9.102,2 +-8.989,2 +-8.876,2 +-8.765,2 +-8.656,2 +-8.546,2 +-8.437,2 +-8.328,2 +-8.22,2 +-8.106,2 +-7.995,2 +-7.886,2 +-7.782,2 +-7.67,2 +-7.563,2 +-7.452,2 +-7.345,2 +-7.236,2 +-7.13,2 +-7.018,2 +-6.911,2 +-6.798,2 +-6.686,2 +-6.579,2 +-6.471,2 +-6.367,2 +-6.256,2 +-6.149,2 +-6.042,2 +-5.934,2 +-5.828,2 +-5.72,2 +-5.616,2 +-5.503,2 +-5.395,2 +-5.286,2 +-5.178,2 +-5.067,2 +-4.961,2 +-4.855,2 +-4.742,2 +-4.635,2 +-4.526,2 +-4.418,2 +-4.304,2 +-4.195,2 +-4.083,2 +-3.971,2 +-3.86,2 +-3.755,2 +-3.641,2 +-3.532,2 +-3.428,2 +-3.32,2 +-3.209,2 +-3.098,2 +-2.994,2 +-2.882,2 +-2.777,2 +-2.665,2 +-2.553,2 +-2.446,2 +-2.337,2 +-2.228,2 +-2.124,2 +-2.018,2 +-1.908,2 +-1.803,2 +-1.693,2 +-1.587,2 +-1.476,2 +-1.37,2 +-1.261,2 +-1.149,2 +-1.037,2 +-0.929,2 +-0.816,2 +-0.713,2 +-0.601,2 +-0.495,2 +-0.387,2 +-0.277,2 +-0.17,2 +-0.066,2 +0.036,2 +0.142,2 +0.253,2 +0.358,2 +0.461,2 +0.571,2 +0.675,2 +0.78,2 +0.884,1 +0.992,1 +1.098,1 +1.205,1 +1.314,1 +1.426,1 +1.538,1 +1.645,1 +1.755,1 +1.864,1 +1.972,1 +2.078,1 +2.188,1 +2.302,1 +2.411,1 +2.522,1 +2.626,1 +2.737,1 +2.838,1 +2.941,1 +3.049,1 +3.156,1 +3.268,1 +3.377,1 +3.49,1 +3.604,1 +3.716,1 +3.825,1 +3.937,1 +4.044,1 +4.154,1 +4.259,1 +4.369,1 +4.479,1 +4.588,1 +4.693,1 +4.8,1 +4.911,1 +5.018,1 +5.127,1 +5.239,1 +5.35,1 +5.457,1 +5.563,1 +5.664,1 +5.767,1 +5.871,1 +5.976,1 +6.086,1 +6.198,1 +6.306,1 +6.413,1 +6.525,1 +6.638,1 +6.747,1 +6.859,1 +6.97,1 +7.081,1 +7.189,1 +7.302,1 +7.411,1 +7.518,1 +7.631,1 +7.74,1 +7.847,1 +7.96,1 +8.071,1 +8.184,1 +8.288,1 +8.396,1 +8.501,1 +8.605,1 +8.711,1 +8.825,1 +8.93,1 +9.034,1 +9.147,1 +9.259,1 +9.367,1 +9.48,1 +9.584,1 +9.687,1 +9.792,1 +9.904,1 +10.011,1 +10.115,1 +10.225,1 +10.327,1 +10.429,1 +10.539,1 +10.64,1 +10.752,1 +10.862,1 +10.967,1 +11.073,1 +11.185,1 +11.297,1 +11.411,1 +11.521,1 +11.631,1 +11.737,1 +11.85,1 +11.955,1 +12.063,1 +12.175,1 +12.288,1 +12.397,1 +12.502,1 +12.616,1 +12.72,1 +12.83,1 +12.934,1 +13.042,1 +13.151,1 +13.264,1 +13.371,1 +13.485,1 +13.587,1 +13.696,1 +13.807,1 +13.912,1 +14.024,1 +14.136,1 +14.237,1 +14.346,1 +14.46,1 +14.571,1 +14.678,1 +14.783,1 +14.885,2 +14.996,2 +15.101,2 +15.203,2 +15.309,2 +15.423,2 +15.533,2 +15.647,2 +15.759,2 +15.87,2 +15.976,2 +16.088,2 +16.196,2 +16.305,2 +16.411,2 +16.523,2 +16.629,2 +16.742,2 +16.854,2 +16.962,2 +17.067,2 +17.181,2 +17.288,2 +17.397,2 +17.509,2 +17.621,2 +17.733,2 +17.84,2 +17.946,2 +18.054,2 +18.163,2 +18.267,2 +18.379,2 +18.492,2 +18.606,2 +18.71,2 +18.819,2 +18.93,2 +19.038,2 +19.14,2 +19.242,2 +19.346,2 +19.455,2 +19.565,2 +19.672,2 +19.786,2 +19.891,2 +20.0,2 +20.103,2 +20.211,2 +20.315,2 +20.42,2 +20.529,2 +20.64,2 +20.747,2 +20.854,2 +20.965,2 +21.073,2 +21.181,2 +21.286,2 +21.396,2 +21.505,2 +21.618,2 +21.73,2 +21.843,2 +21.948,2 +22.055,2 +22.16,2 +22.273,2 +22.385,2 +22.496,2 +22.602,2 +22.712,2 +22.82,2 +22.93,2 +23.042,2 +23.146,2 +23.253,2 +23.36,2 +23.469,2 +23.581,2 +23.69,2 +23.803,2 +23.917,2 +24.025,2 +24.13,2 +24.236,2 +24.348,2 +24.455,2 +24.567,2 +24.678,2 +24.786,2 +24.9,2 +25.013,2 +25.126,2 +25.241,2 +25.353,2 +25.463,2 +25.574,2 +25.686,2 +25.796,2 +25.908,2 +26.017,2 +26.129,2 +26.239,2 +26.35,2 +26.454,2 +26.563,2 +26.672,2 +26.784,2 +26.889,2 +26.994,2 +27.103,2 +27.212,2 +27.325,2 +27.437,2 +27.545,2 +27.658,2 +27.762,2 +27.874,2 +27.978,2 +28.085,2 +28.192,2 +28.301,2 +28.412,2 +28.524,2 +28.638,2 +28.749,2 +28.856,2 +28.966,2 +29.072,2 +29.182,2 +29.29,2 +29.394,2 +29.499,2 +29.607,2 +29.711,2 +29.821,2 +29.929,2 +30.035,2 +30.144,2 +30.255,2 +30.367,2 +30.472,2 +30.585,2 +30.691,2 +30.799,2 +30.911,2 +31.021,2 +31.13,2 +31.238,2 +31.352,2 +31.462,2 +31.576,2 +31.69,2 +31.797,2 +31.909,2 +32.021,2 +32.129,2 +32.233,2 +32.337,2 +32.444,2 +32.555,2 +32.658,2 +32.769,2 +32.881,2 +32.987,2 +33.095,2 +33.209,2 +33.317,2 +33.42,2 +33.532,2 +33.64,2 +33.751,2 +33.86,2 +33.963,2 +34.074,2 +34.179,2 +34.282,2 +34.387,2 +34.495,2 +34.601,2 +34.709,2 +34.821,2 +34.926,2 +35.037,2 +35.141,2 +35.244,2 +35.354,2 +35.462,2 +35.575,2 +35.687,2 +35.796,2 +35.908,2 +36.013,2 +36.12,2 +36.233,2 +36.34,2 +36.446,2 +36.553,2 +36.663,2 +36.771,2 +36.881,2 +36.989,2 +37.095,2 +37.208,2 +37.312,2 +37.424,2 +37.534,2 +37.644,2 +37.748,2 +37.859,2 +37.964,2 +38.068,2 +38.173,2 +38.28,2 +38.392,2 +38.496,2 +38.601,2 +38.712,2 +38.82,2 +38.933,2 +39.045,2 +39.155,2 +39.263,2 +39.376,2 +39.49,2 +39.606,2 +39.72,2 +39.829,2 +39.939,2 +40.042,2 +40.154,2 +40.259,2 +40.365,2 +40.478,2 +40.582,2 +40.692,2 +40.796,2 +40.907,2 +41.015,2 +41.118,2 +41.228,2 +41.338,2 +41.445,2 +41.551,2 +41.662,2 +41.774,2 +41.882,2 +41.993,2 +42.099,2 +42.204,2 +42.312,2 +42.419,2 +42.527,2 +42.637,2 +42.739,2 +42.85,2 +42.961,2 +43.068,2 +43.176,2 +43.278,2 +43.39,2 +43.494,2 +43.599,2 +43.71,2 +43.816,2 +43.92,2 +44.025,2 +44.128,2 +44.233,2 +44.337,2 +44.445,2 +44.552,2 +44.663,2 +44.77,2 +44.881,2 +44.988,2 +45.1,2 +45.203,2 +45.312,2 +45.416,2 +45.523,2 +45.634,2 +45.737,2 diff --git a/evidence/failover-each/7cfd91ddf780/probe.csv b/evidence/failover-each/7cfd91ddf780/probe.csv new file mode 100644 index 00000000..6e766414 --- /dev/null +++ b/evidence/failover-each/7cfd91ddf780/probe.csv @@ -0,0 +1,647 @@ +# date: 2026-11-05T19:18:13.470607+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +t_rel_s,status,latency_ms,instance +-25.852,201,5.2,5bbf259ff136 +-25.744,201,9.9,7cfd91ddf780 +-25.632,201,5.8,5bbf259ff136 +-25.522,201,3.8,7cfd91ddf780 +-25.411,201,12.4,7cfd91ddf780 +-25.288,201,3.8,7cfd91ddf780 +-25.181,201,5.8,5bbf259ff136 +-25.074,201,3.9,5bbf259ff136 +-24.97,201,3.4,5bbf259ff136 +-24.858,201,3.5,5bbf259ff136 +-24.752,201,3.9,7cfd91ddf780 +-24.641,201,6.8,5bbf259ff136 +-24.525,201,4.5,5bbf259ff136 +-24.411,201,4.0,7cfd91ddf780 +-24.302,201,3.2,5bbf259ff136 +-24.192,201,2.8,5bbf259ff136 +-24.087,201,3.0,5bbf259ff136 +-23.974,201,5.0,5bbf259ff136 +-23.859,201,5.2,7cfd91ddf780 +-23.753,201,5.8,5bbf259ff136 +-23.644,201,4.1,5bbf259ff136 +-23.537,201,3.4,5bbf259ff136 +-23.426,201,3.4,5bbf259ff136 +-23.32,201,3.4,5bbf259ff136 +-23.21,201,3.2,5bbf259ff136 +-23.099,201,4.2,5bbf259ff136 +-22.994,201,4.0,7cfd91ddf780 +-22.89,201,3.0,7cfd91ddf780 +-22.777,201,3.0,7cfd91ddf780 +-22.673,201,3.1,5bbf259ff136 +-22.56,201,8.6,7cfd91ddf780 +-22.444,201,4.4,7cfd91ddf780 +-22.33,201,3.6,5bbf259ff136 +-22.223,201,2.7,7cfd91ddf780 +-22.12,201,3.6,7cfd91ddf780 +-22.016,201,2.8,5bbf259ff136 +-21.909,201,7.4,5bbf259ff136 +-21.791,201,4.0,7cfd91ddf780 +-21.678,201,5.2,7cfd91ddf780 +-21.571,201,5.3,5bbf259ff136 +-21.458,201,5.3,7cfd91ddf780 +-21.343,201,5.7,7cfd91ddf780 +-21.227,201,5.0,5bbf259ff136 +-21.12,201,5.7,5bbf259ff136 +-21.009,201,4.8,5bbf259ff136 +-20.894,201,5.2,5bbf259ff136 +-20.787,201,5.9,5bbf259ff136 +-20.676,201,5.1,5bbf259ff136 +-20.569,201,5.8,5bbf259ff136 +-20.461,201,4.7,5bbf259ff136 +-20.355,201,4.5,7cfd91ddf780 +-20.243,201,4.7,5bbf259ff136 +-20.128,201,3.5,5bbf259ff136 +-20.015,201,4.0,5bbf259ff136 +-19.91,201,5.1,5bbf259ff136 +-19.795,201,3.7,5bbf259ff136 +-19.683,201,5.2,5bbf259ff136 +-19.569,201,6.1,7cfd91ddf780 +-19.463,201,6.1,7cfd91ddf780 +-19.35,201,4.4,7cfd91ddf780 +-19.245,201,3.9,5bbf259ff136 +-19.137,201,3.7,5bbf259ff136 +-19.029,201,3.4,5bbf259ff136 +-18.925,201,4.8,7cfd91ddf780 +-18.811,201,5.3,5bbf259ff136 +-18.701,201,5.2,5bbf259ff136 +-18.588,201,4.5,5bbf259ff136 +-18.477,201,5.4,5bbf259ff136 +-18.362,201,5.6,7cfd91ddf780 +-18.25,201,4.7,7cfd91ddf780 +-18.136,201,5.0,7cfd91ddf780 +-18.022,201,5.6,5bbf259ff136 +-17.91,201,5.7,5bbf259ff136 +-17.794,201,5.1,5bbf259ff136 +-17.681,201,5.1,7cfd91ddf780 +-17.575,201,6.1,5bbf259ff136 +-17.461,201,5.2,7cfd91ddf780 +-17.348,201,5.9,7cfd91ddf780 +-17.238,201,4.8,7cfd91ddf780 +-17.126,201,5.6,7cfd91ddf780 +-17.011,201,6.3,5bbf259ff136 +-16.898,201,5.2,5bbf259ff136 +-16.785,201,5.4,5bbf259ff136 +-16.671,201,5.4,7cfd91ddf780 +-16.56,201,4.4,7cfd91ddf780 +-16.453,201,4.8,5bbf259ff136 +-16.342,201,6.1,7cfd91ddf780 +-16.226,201,4.5,7cfd91ddf780 +-16.118,201,4.7,7cfd91ddf780 +-16.009,201,5.1,7cfd91ddf780 +-15.898,201,5.1,7cfd91ddf780 +-15.789,201,5.6,7cfd91ddf780 +-15.673,201,4.5,7cfd91ddf780 +-15.567,201,5.1,7cfd91ddf780 +-15.453,201,5.1,5bbf259ff136 +-15.339,201,6.0,5bbf259ff136 +-15.233,201,5.6,5bbf259ff136 +-15.117,201,3.4,5bbf259ff136 +-15.004,201,10.7,5bbf259ff136 +-14.886,201,6.0,5bbf259ff136 +-14.773,201,5.8,5bbf259ff136 +-14.659,201,5.1,7cfd91ddf780 +-14.548,201,4.0,7cfd91ddf780 +-14.437,201,4.3,7cfd91ddf780 +-14.328,201,3.7,7cfd91ddf780 +-14.217,201,3.2,7cfd91ddf780 +-14.106,201,5.6,7cfd91ddf780 +-13.997,201,5.7,7cfd91ddf780 +-13.886,201,4.7,7cfd91ddf780 +-13.771,201,5.3,5bbf259ff136 +-13.659,201,4.6,5bbf259ff136 +-13.545,201,6.1,5bbf259ff136 +-13.437,201,5.0,5bbf259ff136 +-13.328,201,4.8,5bbf259ff136 +-13.221,201,4.7,5bbf259ff136 +-13.11,201,4.8,5bbf259ff136 +-12.995,201,5.4,5bbf259ff136 +-12.886,201,5.8,5bbf259ff136 +-12.771,201,4.9,7cfd91ddf780 +-12.66,201,5.4,7cfd91ddf780 +-12.553,201,6.2,5bbf259ff136 +-12.438,201,5.3,7cfd91ddf780 +-12.322,201,4.5,5bbf259ff136 +-12.215,201,5.2,7cfd91ddf780 +-12.1,201,5.2,5bbf259ff136 +-11.987,201,4.8,5bbf259ff136 +-11.878,201,5.2,5bbf259ff136 +-11.766,201,5.1,5bbf259ff136 +-11.657,201,4.4,5bbf259ff136 +-11.545,201,6.1,5bbf259ff136 +-11.436,201,4.6,5bbf259ff136 +-11.325,201,5.4,5bbf259ff136 +-11.209,201,4.7,7cfd91ddf780 +-11.098,201,5.5,7cfd91ddf780 +-10.982,201,5.2,5bbf259ff136 +-10.87,201,5.4,5bbf259ff136 +-10.76,201,5.7,5bbf259ff136 +-10.646,201,6.0,5bbf259ff136 +-10.535,201,5.0,5bbf259ff136 +-10.425,201,4.7,5bbf259ff136 +-10.311,201,5.4,7cfd91ddf780 +-10.2,201,4.5,7cfd91ddf780 +-10.085,201,5.1,7cfd91ddf780 +-9.974,201,4.6,7cfd91ddf780 +-9.864,201,3.1,7cfd91ddf780 +-9.753,201,3.5,7cfd91ddf780 +-9.649,201,4.3,7cfd91ddf780 +-9.536,201,4.2,7cfd91ddf780 +-9.427,201,4.4,7cfd91ddf780 +-9.319,201,6.4,7cfd91ddf780 +-9.211,201,6.6,7cfd91ddf780 +-9.101,201,4.1,7cfd91ddf780 +-8.989,201,5.0,7cfd91ddf780 +-8.876,201,5.0,7cfd91ddf780 +-8.765,201,6.1,7cfd91ddf780 +-8.653,201,5.9,5bbf259ff136 +-8.537,201,6.2,5bbf259ff136 +-8.426,201,5.2,5bbf259ff136 +-8.312,201,5.0,5bbf259ff136 +-8.2,201,6.5,7cfd91ddf780 +-8.088,201,4.4,7cfd91ddf780 +-7.973,201,4.4,7cfd91ddf780 +-7.861,201,5.0,5bbf259ff136 +-7.746,201,5.8,5bbf259ff136 +-7.636,201,5.9,5bbf259ff136 +-7.52,201,5.3,7cfd91ddf780 +-7.411,201,5.2,7cfd91ddf780 +-7.298,201,4.9,7cfd91ddf780 +-7.186,201,5.1,7cfd91ddf780 +-7.075,201,5.5,7cfd91ddf780 +-6.966,201,4.8,7cfd91ddf780 +-6.853,201,7.0,5bbf259ff136 +-6.741,201,5.7,5bbf259ff136 +-6.632,201,4.9,5bbf259ff136 +-6.517,201,5.1,5bbf259ff136 +-6.402,201,5.1,7cfd91ddf780 +-6.287,201,4.3,7cfd91ddf780 +-6.176,201,5.5,7cfd91ddf780 +-6.06,201,5.7,5bbf259ff136 +-5.945,201,5.3,5bbf259ff136 +-5.829,201,5.0,7cfd91ddf780 +-5.719,201,4.2,7cfd91ddf780 +-5.604,201,5.3,7cfd91ddf780 +-5.49,201,4.7,5bbf259ff136 +-5.385,201,3.9,7cfd91ddf780 +-5.28,201,4.5,7cfd91ddf780 +-5.171,201,4.4,7cfd91ddf780 +-5.058,201,5.1,7cfd91ddf780 +-4.951,201,4.9,7cfd91ddf780 +-4.845,201,6.2,5bbf259ff136 +-4.729,201,2.9,7cfd91ddf780 +-4.626,201,3.3,5bbf259ff136 +-4.516,201,3.9,5bbf259ff136 +-4.412,201,4.6,5bbf259ff136 +-4.303,201,5.2,5bbf259ff136 +-4.194,201,4.7,7cfd91ddf780 +-4.089,201,4.7,7cfd91ddf780 +-3.974,201,5.0,7cfd91ddf780 +-3.862,201,5.4,5bbf259ff136 +-3.755,201,6.0,7cfd91ddf780 +-3.641,201,6.5,5bbf259ff136 +-3.532,201,5.1,5bbf259ff136 +-3.418,201,4.7,5bbf259ff136 +-3.305,201,4.8,5bbf259ff136 +-3.195,201,5.7,5bbf259ff136 +-3.085,201,4.4,5bbf259ff136 +-2.975,201,5.2,5bbf259ff136 +-2.862,201,4.9,7cfd91ddf780 +-2.748,201,4.5,7cfd91ddf780 +-2.637,201,4.6,7cfd91ddf780 +-2.529,201,6.1,7cfd91ddf780 +-2.413,201,5.2,5bbf259ff136 +-2.302,201,5.6,5bbf259ff136 +-2.187,201,4.9,5bbf259ff136 +-2.072,201,5.5,7cfd91ddf780 +-1.956,201,5.1,7cfd91ddf780 +-1.849,201,5.2,7cfd91ddf780 +-1.737,201,5.4,7cfd91ddf780 +-1.625,201,5.3,7cfd91ddf780 +-1.51,201,5.3,5bbf259ff136 +-1.404,201,6.3,7cfd91ddf780 +-1.291,201,5.5,5bbf259ff136 +-1.185,201,5.8,7cfd91ddf780 +-1.078,201,6.7,7cfd91ddf780 +-0.964,201,7.0,5bbf259ff136 +-0.855,201,5.7,7cfd91ddf780 +-0.748,201,4.7,7cfd91ddf780 +-0.642,201,5.7,7cfd91ddf780 +-0.526,201,5.6,7cfd91ddf780 +-0.42,201,5.1,7cfd91ddf780 +-0.309,201,5.5,7cfd91ddf780 +-0.199,201,5.0,7cfd91ddf780 +-0.088,201,5.2,7cfd91ddf780 +0.019,201,4.0,7cfd91ddf780 +0.133,201,3.5,7cfd91ddf780 +0.241,201,3.7,7cfd91ddf780 +0.352,201,6.6,7cfd91ddf780 +0.46,201,5.2,7cfd91ddf780 +0.569,201,4.6,7cfd91ddf780 +0.675,201,5.2,5bbf259ff136 +0.79,201,4.4,7cfd91ddf780 +0.896,201,4.5,5bbf259ff136 +1.005,201,5.1,5bbf259ff136 +1.118,201,5.4,5bbf259ff136 +1.233,201,3.4,5bbf259ff136 +1.343,201,4.1,5bbf259ff136 +1.457,201,4.8,5bbf259ff136 +1.565,201,4.2,5bbf259ff136 +1.679,201,4.6,5bbf259ff136 +1.785,201,6.0,5bbf259ff136 +1.895,201,6.1,5bbf259ff136 +2.011,201,4.3,5bbf259ff136 +2.117,201,6.7,5bbf259ff136 +2.234,201,4.1,5bbf259ff136 +2.342,201,5.5,5bbf259ff136 +2.457,201,6.2,5bbf259ff136 +2.57,201,3.6,5bbf259ff136 +2.675,201,4.6,5bbf259ff136 +2.78,201,4.0,5bbf259ff136 +2.892,201,3.2,5bbf259ff136 +2.995,201,3.1,5bbf259ff136 +3.106,201,6.7,5bbf259ff136 +3.218,201,6.4,5bbf259ff136 +3.334,201,6.6,5bbf259ff136 +3.451,201,6.3,5bbf259ff136 +3.563,201,5.6,5bbf259ff136 +3.669,201,7.1,5bbf259ff136 +3.78,201,4.6,5bbf259ff136 +3.887,201,5.8,5bbf259ff136 +4.003,201,6.2,5bbf259ff136 +4.12,201,5.4,5bbf259ff136 +4.234,201,5.0,5bbf259ff136 +4.348,201,6.5,5bbf259ff136 +4.458,201,6.0,5bbf259ff136 +4.564,201,5.9,5bbf259ff136 +4.674,201,5.4,5bbf259ff136 +4.787,201,7.3,5bbf259ff136 +4.9,201,5.7,5bbf259ff136 +5.014,201,5.3,5bbf259ff136 +5.126,201,5.2,5bbf259ff136 +5.241,201,5.2,5bbf259ff136 +5.352,201,3.0,5bbf259ff136 +5.458,201,3.6,5bbf259ff136 +5.566,201,2.8,5bbf259ff136 +5.672,201,3.0,5bbf259ff136 +5.776,201,4.3,5bbf259ff136 +5.882,201,4.2,5bbf259ff136 +5.99,201,5.6,5bbf259ff136 +6.097,201,3.8,5bbf259ff136 +6.204,201,5.5,5bbf259ff136 +6.31,201,5.2,5bbf259ff136 +6.421,201,3.4,5bbf259ff136 +6.529,201,4.4,5bbf259ff136 +6.636,201,3.7,5bbf259ff136 +6.747,201,4.6,5bbf259ff136 +6.861,201,4.7,5bbf259ff136 +6.974,201,3.9,5bbf259ff136 +7.081,201,5.7,5bbf259ff136 +7.189,201,6.0,5bbf259ff136 +7.302,201,4.3,5bbf259ff136 +7.412,201,4.8,5bbf259ff136 +7.522,201,4.8,5bbf259ff136 +7.631,201,4.5,5bbf259ff136 +7.739,201,6.3,5bbf259ff136 +7.854,201,3.7,5bbf259ff136 +7.959,201,6.1,5bbf259ff136 +8.075,201,6.3,5bbf259ff136 +8.191,201,5.6,5bbf259ff136 +8.306,201,6.0,5bbf259ff136 +8.422,201,5.4,5bbf259ff136 +8.537,201,5.8,5bbf259ff136 +8.651,201,5.3,5bbf259ff136 +8.758,201,6.5,5bbf259ff136 +8.869,201,5.5,5bbf259ff136 +8.977,201,7.0,5bbf259ff136 +9.094,201,4.9,5bbf259ff136 +9.202,201,5.9,5bbf259ff136 +9.313,201,5.8,5bbf259ff136 +9.421,201,4.4,5bbf259ff136 +9.526,201,6.7,5bbf259ff136 +9.64,201,7.0,5bbf259ff136 +9.753,201,7.7,5bbf259ff136 +9.869,201,5.7,5bbf259ff136 +9.979,201,5.9,5bbf259ff136 +10.086,201,6.6,5bbf259ff136 +10.194,201,4.8,5bbf259ff136 +10.309,201,4.9,5bbf259ff136 +10.419,201,3.7,5bbf259ff136 +10.529,201,4.7,5bbf259ff136 +10.64,201,5.3,5bbf259ff136 +10.751,201,4.1,5bbf259ff136 +10.862,201,4.6,5bbf259ff136 +10.968,201,6.9,5bbf259ff136 +11.084,201,5.0,5bbf259ff136 +11.191,201,4.0,5bbf259ff136 +11.296,201,7.0,5bbf259ff136 +11.411,201,6.1,5bbf259ff136 +11.527,201,6.0,5bbf259ff136 +11.641,201,4.6,5bbf259ff136 +11.753,201,4.7,5bbf259ff136 +11.868,201,5.1,5bbf259ff136 +11.976,201,6.4,5bbf259ff136 +12.088,201,4.8,5bbf259ff136 +12.199,201,3.8,5bbf259ff136 +12.313,201,4.7,5bbf259ff136 +12.424,201,4.9,5bbf259ff136 +12.535,201,4.9,5bbf259ff136 +12.65,201,5.2,5bbf259ff136 +12.762,201,4.6,5bbf259ff136 +12.869,201,4.6,5bbf259ff136 +12.977,201,3.8,5bbf259ff136 +13.089,201,3.6,5bbf259ff136 +13.196,201,5.8,5bbf259ff136 +13.304,201,4.5,5bbf259ff136 +13.413,201,4.0,5bbf259ff136 +13.526,201,5.2,5bbf259ff136 +13.633,201,5.3,5bbf259ff136 +13.747,201,3.9,5bbf259ff136 +13.859,201,3.8,5bbf259ff136 +13.969,201,3.9,5bbf259ff136 +14.076,201,3.7,5bbf259ff136 +14.184,201,3.5,5bbf259ff136 +14.296,201,5.1,5bbf259ff136 +14.409,201,4.4,5bbf259ff136 +14.519,201,7.0,5bbf259ff136 +14.632,201,6.5,5bbf259ff136 +14.746,201,6.7,5bbf259ff136 +14.855,201,5.9,5bbf259ff136 +14.968,201,11.9,7cfd91ddf780 +15.088,201,5.6,5bbf259ff136 +15.203,201,4.0,5bbf259ff136 +15.31,201,4.9,5bbf259ff136 +15.423,201,6.8,5bbf259ff136 +15.538,201,4.5,7cfd91ddf780 +15.646,201,5.7,7cfd91ddf780 +15.758,201,7.3,7cfd91ddf780 +15.868,201,8.8,7cfd91ddf780 +15.987,201,3.4,5bbf259ff136 +16.1,201,4.6,5bbf259ff136 +16.207,201,4.9,5bbf259ff136 +16.319,201,4.9,5bbf259ff136 +16.432,201,4.5,5bbf259ff136 +16.538,201,4.7,5bbf259ff136 +16.651,201,6.0,5bbf259ff136 +16.763,201,4.9,5bbf259ff136 +16.874,201,4.4,5bbf259ff136 +16.979,201,4.1,5bbf259ff136 +17.09,201,5.1,5bbf259ff136 +17.197,201,5.2,5bbf259ff136 +17.312,201,5.9,5bbf259ff136 +17.426,201,5.1,5bbf259ff136 +17.541,201,5.4,7cfd91ddf780 +17.652,201,5.2,7cfd91ddf780 +17.761,201,5.1,7cfd91ddf780 +17.877,201,6.2,5bbf259ff136 +17.993,201,6.3,5bbf259ff136 +18.109,201,6.2,7cfd91ddf780 +18.218,201,5.0,7cfd91ddf780 +18.334,201,5.3,7cfd91ddf780 +18.442,201,5.2,7cfd91ddf780 +18.55,201,5.5,7cfd91ddf780 +18.663,201,5.2,7cfd91ddf780 +18.775,201,5.8,5bbf259ff136 +18.891,201,4.3,5bbf259ff136 +19.006,201,11.6,7cfd91ddf780 +19.127,201,4.9,5bbf259ff136 +19.241,201,6.1,5bbf259ff136 +19.35,201,5.0,5bbf259ff136 +19.463,201,4.9,5bbf259ff136 +19.571,201,5.1,5bbf259ff136 +19.682,201,4.7,5bbf259ff136 +19.795,201,5.0,5bbf259ff136 +19.91,201,4.4,7cfd91ddf780 +20.02,201,6.2,7cfd91ddf780 +20.129,201,5.4,7cfd91ddf780 +20.244,201,5.8,7cfd91ddf780 +20.354,201,5.2,7cfd91ddf780 +20.468,201,5.6,5bbf259ff136 +20.579,201,7.0,5bbf259ff136 +20.686,201,4.7,7cfd91ddf780 +20.793,201,4.9,7cfd91ddf780 +20.901,201,5.0,7cfd91ddf780 +21.009,201,3.9,7cfd91ddf780 +21.121,201,4.9,7cfd91ddf780 +21.229,201,5.9,7cfd91ddf780 +21.337,201,4.9,7cfd91ddf780 +21.446,201,5.4,7cfd91ddf780 +21.562,201,6.4,7cfd91ddf780 +21.671,201,6.0,7cfd91ddf780 +21.787,201,5.8,5bbf259ff136 +21.893,201,4.9,7cfd91ddf780 +22.007,201,5.0,7cfd91ddf780 +22.116,201,5.8,5bbf259ff136 +22.222,201,5.2,7cfd91ddf780 +22.329,201,4.9,7cfd91ddf780 +22.438,201,5.8,7cfd91ddf780 +22.553,201,6.3,7cfd91ddf780 +22.669,201,5.5,5bbf259ff136 +22.785,201,6.3,7cfd91ddf780 +22.896,201,5.5,5bbf259ff136 +23.006,201,6.2,7cfd91ddf780 +23.115,201,6.6,5bbf259ff136 +23.232,201,5.3,7cfd91ddf780 +23.347,201,6.7,5bbf259ff136 +23.461,201,6.8,5bbf259ff136 +23.578,201,4.1,7cfd91ddf780 +23.687,201,6.1,7cfd91ddf780 +23.802,201,6.8,7cfd91ddf780 +23.917,201,6.0,5bbf259ff136 +24.033,201,3.2,5bbf259ff136 +24.145,201,3.9,5bbf259ff136 +24.256,201,4.2,7cfd91ddf780 +24.363,201,4.9,5bbf259ff136 +24.468,201,5.0,5bbf259ff136 +24.577,201,4.5,5bbf259ff136 +24.687,201,5.2,7cfd91ddf780 +24.797,201,5.3,5bbf259ff136 +24.909,201,4.7,5bbf259ff136 +25.021,201,5.0,5bbf259ff136 +25.126,201,7.2,5bbf259ff136 +25.239,201,6.0,5bbf259ff136 +25.35,201,5.8,5bbf259ff136 +25.463,201,6.2,5bbf259ff136 +25.573,201,5.5,5bbf259ff136 +25.687,201,5.2,7cfd91ddf780 +25.797,201,5.8,7cfd91ddf780 +25.912,201,5.2,7cfd91ddf780 +26.021,201,4.8,7cfd91ddf780 +26.131,201,3.5,7cfd91ddf780 +26.235,201,3.7,5bbf259ff136 +26.349,201,5.5,7cfd91ddf780 +26.465,201,4.8,7cfd91ddf780 +26.57,201,4.5,7cfd91ddf780 +26.679,201,4.2,7cfd91ddf780 +26.793,201,4.8,7cfd91ddf780 +26.903,201,4.9,7cfd91ddf780 +27.012,201,5.4,7cfd91ddf780 +27.121,201,5.7,7cfd91ddf780 +27.227,201,5.0,7cfd91ddf780 +27.342,201,5.4,7cfd91ddf780 +27.452,201,5.1,7cfd91ddf780 +27.565,201,4.8,7cfd91ddf780 +27.675,201,5.0,7cfd91ddf780 +27.79,201,5.4,5bbf259ff136 +27.904,201,5.8,5bbf259ff136 +28.014,201,6.5,5bbf259ff136 +28.131,201,5.5,7cfd91ddf780 +28.246,201,5.7,5bbf259ff136 +28.354,201,5.8,7cfd91ddf780 +28.467,201,6.2,5bbf259ff136 +28.581,201,5.6,5bbf259ff136 +28.69,201,4.3,5bbf259ff136 +28.795,201,5.1,7cfd91ddf780 +28.907,201,5.1,5bbf259ff136 +29.015,201,6.1,7cfd91ddf780 +29.123,201,6.4,7cfd91ddf780 +29.237,201,3.9,5bbf259ff136 +29.349,201,4.2,5bbf259ff136 +29.455,201,7.7,5bbf259ff136 +29.569,201,7.2,5bbf259ff136 +29.679,201,6.3,5bbf259ff136 +29.794,201,7.5,5bbf259ff136 +29.91,201,5.8,7cfd91ddf780 +30.022,201,6.9,7cfd91ddf780 +30.136,201,6.7,5bbf259ff136 +30.252,201,6.3,5bbf259ff136 +30.367,201,6.1,7cfd91ddf780 +30.479,201,6.8,7cfd91ddf780 +30.595,201,4.9,7cfd91ddf780 +30.703,201,5.1,7cfd91ddf780 +30.813,201,4.7,7cfd91ddf780 +30.924,201,5.3,7cfd91ddf780 +31.032,201,5.1,7cfd91ddf780 +31.143,201,4.5,7cfd91ddf780 +31.257,201,4.8,5bbf259ff136 +31.371,201,3.3,5bbf259ff136 +31.477,201,4.6,5bbf259ff136 +31.586,201,3.6,5bbf259ff136 +31.699,201,4.4,5bbf259ff136 +31.813,201,4.8,5bbf259ff136 +31.927,201,4.6,7cfd91ddf780 +32.042,201,5.3,7cfd91ddf780 +32.158,201,5.0,5bbf259ff136 +32.27,201,4.2,5bbf259ff136 +32.378,201,5.6,5bbf259ff136 +32.487,201,5.7,5bbf259ff136 +32.603,201,4.3,5bbf259ff136 +32.712,201,6.5,5bbf259ff136 +32.828,201,6.7,7cfd91ddf780 +32.945,201,5.3,7cfd91ddf780 +33.054,201,5.8,7cfd91ddf780 +33.168,201,6.3,5bbf259ff136 +33.279,201,8.0,5bbf259ff136 +33.393,201,5.6,5bbf259ff136 +33.503,201,4.7,5bbf259ff136 +33.616,201,6.1,7cfd91ddf780 +33.731,201,4.7,7cfd91ddf780 +33.845,201,7.1,7cfd91ddf780 +33.954,201,5.1,7cfd91ddf780 +34.069,201,5.8,5bbf259ff136 +34.179,201,5.2,5bbf259ff136 +34.288,201,4.7,5bbf259ff136 +34.399,201,3.1,5bbf259ff136 +34.512,201,3.5,5bbf259ff136 +34.625,201,4.0,5bbf259ff136 +34.733,201,4.6,5bbf259ff136 +34.848,201,5.8,7cfd91ddf780 +34.957,201,4.2,7cfd91ddf780 +35.062,201,6.9,5bbf259ff136 +35.169,201,5.2,5bbf259ff136 +35.282,201,6.0,5bbf259ff136 +35.395,201,5.3,5bbf259ff136 +35.503,201,5.2,5bbf259ff136 +35.614,201,5.8,5bbf259ff136 +35.72,201,4.9,5bbf259ff136 +35.83,201,6.2,5bbf259ff136 +35.938,201,5.5,5bbf259ff136 +36.05,201,6.1,5bbf259ff136 +36.159,201,5.0,5bbf259ff136 +36.273,201,5.2,5bbf259ff136 +36.387,201,6.5,7cfd91ddf780 +36.495,201,8.3,5bbf259ff136 +36.611,201,3.5,7cfd91ddf780 +36.716,201,4.1,7cfd91ddf780 +36.823,201,7.0,5bbf259ff136 +36.937,201,4.8,7cfd91ddf780 +37.046,201,6.6,7cfd91ddf780 +37.16,201,5.2,7cfd91ddf780 +37.271,201,4.7,7cfd91ddf780 +37.378,201,5.3,7cfd91ddf780 +37.487,201,6.1,7cfd91ddf780 +37.596,201,4.5,7cfd91ddf780 +37.71,201,4.0,7cfd91ddf780 +37.821,201,5.1,7cfd91ddf780 +37.927,201,4.4,7cfd91ddf780 +38.037,201,6.3,7cfd91ddf780 +38.145,201,6.1,5bbf259ff136 +38.259,201,5.1,7cfd91ddf780 +38.373,201,8.5,7cfd91ddf780 +38.485,201,5.3,7cfd91ddf780 +38.592,201,4.8,7cfd91ddf780 +38.706,201,4.2,7cfd91ddf780 +38.82,201,4.7,5bbf259ff136 +38.933,201,5.9,5bbf259ff136 +39.041,201,6.6,5bbf259ff136 +39.155,201,6.8,7cfd91ddf780 +39.269,201,5.0,7cfd91ddf780 +39.379,201,5.5,7cfd91ddf780 +39.49,201,11.5,7cfd91ddf780 +39.605,201,6.4,5bbf259ff136 +39.72,201,5.7,5bbf259ff136 +39.828,201,6.5,5bbf259ff136 +39.939,201,6.0,5bbf259ff136 +40.054,201,5.0,5bbf259ff136 +40.161,201,4.9,5bbf259ff136 +40.27,201,5.6,5bbf259ff136 +40.377,201,6.1,7cfd91ddf780 +40.493,201,4.2,5bbf259ff136 +40.604,201,6.2,5bbf259ff136 +40.71,201,4.2,5bbf259ff136 +40.817,201,4.2,5bbf259ff136 +40.927,201,5.0,5bbf259ff136 +41.039,201,5.1,5bbf259ff136 +41.154,201,5.1,5bbf259ff136 +41.264,201,4.9,5bbf259ff136 +41.372,201,5.1,5bbf259ff136 +41.487,201,6.8,7cfd91ddf780 +41.597,201,3.7,7cfd91ddf780 +41.711,201,3.9,7cfd91ddf780 +41.824,201,4.3,7cfd91ddf780 +41.939,201,5.0,5bbf259ff136 +42.049,201,6.2,5bbf259ff136 +42.16,201,5.7,5bbf259ff136 +42.27,201,3.6,5bbf259ff136 +42.378,201,3.7,5bbf259ff136 +42.492,201,4.5,5bbf259ff136 +42.599,201,4.3,5bbf259ff136 +42.711,201,4.0,5bbf259ff136 +42.821,201,6.6,5bbf259ff136 +42.937,201,4.8,7cfd91ddf780 +43.048,201,6.1,7cfd91ddf780 +43.16,201,3.3,7cfd91ddf780 +43.265,201,3.2,5bbf259ff136 +43.37,201,3.5,5bbf259ff136 +43.482,201,3.4,5bbf259ff136 +43.59,201,3.1,5bbf259ff136 +43.698,201,7.8,5bbf259ff136 +43.816,201,7.4,7cfd91ddf780 +43.928,201,5.0,7cfd91ddf780 +44.036,201,5.0,7cfd91ddf780 +44.149,201,5.4,7cfd91ddf780 +44.264,201,6.0,7cfd91ddf780 +44.371,201,5.1,7cfd91ddf780 +44.481,201,4.8,7cfd91ddf780 +44.595,201,3.4,7cfd91ddf780 +44.7,201,3.2,7cfd91ddf780 +44.812,201,2.8,7cfd91ddf780 +44.925,201,4.3,7cfd91ddf780 +45.034,201,5.7,5bbf259ff136 +45.145,201,8.0,7cfd91ddf780 +45.26,201,8.5,5bbf259ff136 +45.369,201,8.2,7cfd91ddf780 +45.487,201,11.7,5bbf259ff136 +45.607,201,11.0,7cfd91ddf780 +45.727,201,4.3,5bbf259ff136 diff --git a/evidence/failover-each/ff81f93e8b0d/result.json b/evidence/failover-each/7cfd91ddf780/result.json similarity index 59% rename from evidence/failover-each/ff81f93e8b0d/result.json rename to evidence/failover-each/7cfd91ddf780/result.json index 19b00d49..a1121c39 100644 --- a/evidence/failover-each/ff81f93e8b0d/result.json +++ b/evidence/failover-each/7cfd91ddf780/result.json @@ -1,34 +1,42 @@ -{ - "label": "ff81f93e8b0d", +{"date": "2026-11-05T19:18:13.470607+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", + "label": "7cfd91ddf780", "action": [ "sh", "-c", "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' api && { scripts/stand-usage.sh & sleep 10; wait; } && scripts/restore.sh" ], "action_exit": 0, - "action_seconds": 15.296, + "action_seconds": 15.204, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 1, - "detection_seconds": 1.405, - "rejoin_seconds": 14.834, + "lb_samples_missing": 0, + "detection_seconds": 0.884, + "rejoin_seconds": 14.885, "probe": { - "attempts": 646, + "attempts": 645, "failed_attempts": 0, "first_failure_s": null, + "first_success_s": 0.023, "recovery_s": 0.0, "instances_before": { - "ff81f93e8b0d": 116, - "fe75c6c37e5a": 117 + "5bbf259ff136": 118, + "7cfd91ddf780": 115 }, "instances_during_outage": { - "fe75c6c37e5a": 121 + "5bbf259ff136": 126 }, "instances_after": { - "ff81f93e8b0d": 137, - "fe75c6c37e5a": 138 + "5bbf259ff136": 142, + "7cfd91ddf780": 132 } }, + "history": { + "confirmed": 645, + "stored": 645, + "exact_match": true, + "unique": true + }, "lb_counters": { "wretr": 0, "wredis": 0, @@ -42,16 +50,19 @@ "ok_rps": 100.02, "failed_ratio": 0, "dropped_iterations": 0, - "p50_ms": 1.6, - "p95_ms": 3.92, - "p99_ms": 7.31, + "p50_ms": 2.97, + "p95_ms": 3.86, + "p99_ms": 4.47, "exit": 0, "rate": 100 }, "checks": { "action_ok": true, "load_errors_ok": true, + "load_delivered": true, + "lb_observed": true, "write_recovery_ok": true, + "history_preserved": true, "detected": true, "target_excluded": true, "rejoined": true diff --git a/evidence/failover-each/fe75c6c37e5a/action.txt b/evidence/failover-each/fe75c6c37e5a/action.txt deleted file mode 100644 index 229a8c5c..00000000 --- a/evidence/failover-each/fe75c6c37e5a/action.txt +++ /dev/null @@ -1,14 +0,0 @@ -ff81f93e8b0d api Up About a minute (healthy) -fe75c6c37e5a api Exited (0) Less than a second ago -02:49:51 database bytes,messages: 12867251,9053 -mtusi2026lab3-api-1 cpu=22.79% mem=142.2MiB / 512MiB -mtusi2026lab3-lb-1 cpu=2.61% mem=20.39MiB / 128MiB -mtusi2026lab3-postgres-1 cpu=4.10% mem=49.27MiB / 1GiB -mtusi2026lab3-valkey-1 cpu=1.32% mem=13.1MiB / 128MiB - Container mtusi2026lab3-api-1 Running - Container mtusi2026lab3-api-2 Starting - Container mtusi2026lab3-api-2 Started - Container mtusi2026lab3-api-1 Waiting - Container mtusi2026lab3-api-2 Waiting - Container mtusi2026lab3-api-1 Healthy - Container mtusi2026lab3-api-2 Healthy diff --git a/evidence/failover-each/fe75c6c37e5a/k6/k6.txt b/evidence/failover-each/fe75c6c37e5a/k6/k6.txt deleted file mode 100644 index c0e3e690..00000000 --- a/evidence/failover-each/fe75c6c37e5a/k6/k6.txt +++ /dev/null @@ -1,2 +0,0 @@ -time="2026-10-30T02:49:24Z" level=info msg=LOAD-START source=console -measure: 6001 requests, failed ratio 0 diff --git a/evidence/failover-each/fe75c6c37e5a/k6/summary.json b/evidence/failover-each/fe75c6c37e5a/k6/summary.json deleted file mode 100644 index 2c07b91d..00000000 --- a/evidence/failover-each/fe75c6c37e5a/k6/summary.json +++ /dev/null @@ -1,309 +0,0 @@ -{ - "state": { - "isStdOutTTY": false, - "isStdErrTTY": false, - "testRunDurationMs": 70503.117005 - }, - "metrics": { - "vus": { - "values": { - "value": 0, - "min": 0, - "max": 3 - }, - "type": "gauge", - "contains": "default" - }, - "http_req_duration{scenario:measure,name:post_message}": { - "type": "trend", - "contains": "time", - "values": { - "min": 0.862875, - "med": 1.548979, - "p(90)": 2.9327250000000005, - "p(95)": 3.5690767499999994, - "p(99)": 6.153909249999998, - "max": 28.063083, - "avg": 1.874677454863969 - } - }, - "http_req_duration{scenario:measure}": { - "type": "trend", - "contains": "time", - "values": { - "p(95)": 3.618333, - "p(99)": 6.23825, - "max": 36.577708, - "avg": 1.9048608050324902, - "min": 0.862875, - "med": 1.566209, - "p(90)": 2.972667 - } - }, - "dropped_iterations{scenario:measure}": { - "contains": "default", - "values": { - "count": 0, - "rate": 0 - }, - "type": "counter" - }, - "http_req_failed": { - "values": { - "rate": 0, - "passes": 0, - "fails": 7052 - }, - "type": "rate", - "contains": "default" - }, - "iteration_duration": { - "type": "trend", - "contains": "time", - "values": { - "p(90)": 3.0199375, - "p(95)": 3.6165499999999997, - "p(99)": 6.239641759999998, - "max": 36.820709, - "avg": 1.9613262119394446, - "min": 0.921, - "med": 1.6351040000000001 - } - }, - "vus_max": { - "type": "gauge", - "contains": "default", - "values": { - "min": 20, - "max": 20, - "value": 20 - } - }, - "http_req_tls_handshaking": { - "type": "trend", - "contains": "time", - "values": { - "avg": 0, - "min": 0, - "med": 0, - "p(90)": 0, - "p(95)": 0, - "p(99)": 0, - "max": 0 - } - }, - "http_reqs": { - "type": "counter", - "contains": "default", - "values": { - "count": 7052, - "rate": 100.02394645189774 - } - }, - "dropped_iterations": { - "type": "counter", - "contains": "default", - "values": { - "count": 0, - "rate": 0 - } - }, - "http_req_sending": { - "type": "trend", - "contains": "time", - "values": { - "avg": 0.013893221922858738, - "min": 0.00275, - "med": 0.011458, - "p(90)": 0.017583, - "p(95)": 0.021435449999999988, - "p(99)": 0.054310919999999985, - "max": 1.451542 - } - }, - "http_req_receiving": { - "contains": "time", - "values": { - "p(95)": 0.05310244999999999, - "p(99)": 0.08932850999999996, - "max": 3.4675, - "avg": 0.027733872376630664, - "min": 0.006167, - "med": 0.023041, - "p(90)": 0.0425839 - }, - "type": "trend" - }, - "data_received": { - "type": "counter", - "contains": "data", - "values": { - "count": 16604156, - "rate": 235509.53071794618 - } - }, - "http_req_duration{scenario:measure,name:get_history}": { - "type": "trend", - "contains": "time", - "values": { - "p(95)": 3.822308, - "p(99)": 6.884161999999998, - "max": 36.577708, - "avg": 2.0323191296779815, - "min": 0.998708, - "med": 1.624583, - "p(90)": 3.0656336 - } - }, - "checks": { - "type": "rate", - "contains": "default", - "values": { - "rate": 1, - "passes": 7002, - "fails": 0 - } - }, - "http_req_connecting": { - "type": "trend", - "contains": "time", - "values": { - "max": 0.188833, - "avg": 0.00041799602949517856, - "min": 0, - "med": 0, - "p(90)": 0, - "p(95)": 0, - "p(99)": 0 - } - }, - "http_req_failed{scenario:measure}": { - "type": "rate", - "contains": "default", - "values": { - "rate": 0, - "passes": 0, - "fails": 6001 - } - }, - "iterations{scenario:measure}": { - "type": "counter", - "contains": "default", - "values": { - "count": 6001, - "rate": 85.11680412051025 - } - }, - "iterations": { - "type": "counter", - "contains": "default", - "values": { - "count": 7002, - "rate": 99.3147579489773 - } - }, - "http_req_blocked": { - "type": "trend", - "contains": "time", - "values": { - "p(90)": 0.003291, - "p(95)": 0.004041, - "p(99)": 0.006208, - "max": 0.494459, - "avg": 0.0027004892229154963, - "min": 0.000666, - "med": 0.001875 - } - }, - "http_req_duration": { - "type": "trend", - "contains": "time", - "values": { - "p(95)": 3.60760595, - "p(99)": 8.653849589999998, - "max": 36.577708, - "avg": 1.926197078417462, - "min": 0.862875, - "med": 1.5504995, - "p(90)": 2.94885 - } - }, - "http_reqs{scenario:measure}": { - "type": "counter", - "contains": "default", - "values": { - "count": 6001, - "rate": 85.11680412051025 - } - }, - "http_req_duration{expected_response:true}": { - "type": "trend", - "contains": "time", - "values": { - "avg": 1.926197078417462, - "min": 0.862875, - "med": 1.5504995, - "p(90)": 2.94885, - "p(95)": 3.60760595, - "p(99)": 8.653849589999998, - "max": 36.577708 - } - }, - "http_req_waiting": { - "type": "trend", - "contains": "time", - "values": { - "min": 0.842875, - "med": 1.514396, - "p(90)": 2.8950378, - "p(95)": 3.5574022999999992, - "p(99)": 8.57573916, - "max": 36.512667, - "avg": 1.8845699841179808 - } - }, - "data_sent": { - "values": { - "count": 2058898, - "rate": 29202.935805717436 - }, - "type": "counter", - "contains": "data" - } - }, - "root_group": { - "name": "", - "path": "", - "id": "d41d8cd98f00b204e9800998ecf8427e", - "groups": [], - "checks": [ - { - "name": "message 201", - "path": "::message 201", - "id": "6445aff337e1ade0bc10402fbac79882", - "passes": 5661, - "fails": 0 - }, - { - "id": "fe27b3b8edd292c7f2170a6357b087d0", - "passes": 1341, - "fails": 0, - "name": "history 200", - "path": "::history 200" - } - ] - }, - "options": { - "summaryTrendStats": [ - "avg", - "min", - "med", - "p(90)", - "p(95)", - "p(99)", - "max" - ], - "summaryTimeUnit": "", - "noColor": false - } -} \ No newline at end of file diff --git a/evidence/failover-each/fe75c6c37e5a/lb_up.csv b/evidence/failover-each/fe75c6c37e5a/lb_up.csv deleted file mode 100644 index 1a11f82f..00000000 --- a/evidence/failover-each/fe75c6c37e5a/lb_up.csv +++ /dev/null @@ -1,653 +0,0 @@ -t_rel_s,up --25.746,2 --25.634,2 --25.532,2 --25.424,2 --25.309,2 --25.199,2 --25.092,2 --24.978,2 --24.866,2 --24.753,2 --24.647,2 --24.54,2 --24.43,2 --24.322,2 --24.216,2 --24.106,2 --23.997,2 --23.89,2 --23.784,2 --23.672,2 --23.56,2 --23.454,2 --23.34,2 --23.231,2 --23.118,2 --23.006,2 --22.899,2 --22.793,2 --22.687,2 --22.58,2 --22.476,2 --22.364,2 --22.254,2 --22.141,2 --22.029,2 --21.922,2 --21.817,2 --21.705,2 --21.601,2 --21.489,2 --21.378,2 --21.271,2 --21.161,2 --21.057,2 --20.952,2 --20.836,2 --20.728,2 --20.621,2 --20.507,2 --20.397,2 --20.287,2 --20.174,2 --20.061,2 --19.951,2 --19.841,2 --19.732,2 --19.619,2 --19.508,2 --19.397,2 --19.289,2 --19.182,2 --19.073,2 --18.964,2 --18.856,2 --18.749,2 --18.646,2 --18.538,2 --18.431,2 --18.326,2 --18.215,2 --18.111,2 --18.004,2 --17.892,2 --17.787,2 --17.675,2 --17.564,2 --17.449,2 --17.339,2 --17.226,2 --17.118,2 --17.008,2 --16.898,2 --16.795,2 --16.687,2 --16.58,2 --16.469,2 --16.36,2 --16.254,2 --16.149,2 --16.039,2 --15.934,2 --15.822,2 --15.719,2 --15.61,2 --15.502,2 --15.393,2 --15.285,2 --15.179,2 --15.068,2 --14.956,2 --14.852,2 --14.743,2 --14.634,2 --14.521,2 --14.408,2 --14.301,2 --14.19,2 --14.084,2 --13.979,2 --13.868,2 --13.761,2 --13.654,2 --13.551,2 --13.439,2 --13.327,2 --13.222,2 --13.113,2 --13.006,2 --12.899,2 --12.786,2 --12.675,2 --12.57,2 --12.462,2 --12.351,2 --12.24,2 --12.135,2 --12.023,2 --11.913,2 --11.802,2 --11.69,2 --11.582,2 --11.472,2 --11.36,2 --11.247,2 --11.141,2 --11.035,2 --10.925,2 --10.813,2 --10.707,2 --10.595,2 --10.49,2 --10.381,2 --10.274,2 --10.166,2 --10.057,2 --9.95,2 --9.841,2 --9.735,2 --9.627,2 --9.516,2 --9.412,2 --9.309,2 --9.196,2 --9.082,2 --8.968,2 --8.864,2 --8.757,2 --8.649,2 --8.541,2 --8.434,2 --8.329,2 --8.224,2 --8.114,2 --8.003,2 --7.896,2 --7.783,2 --7.67,2 --7.559,2 --7.452,2 --7.341,2 --7.233,2 --7.125,2 --7.014,2 --6.904,2 --6.794,2 --6.682,2 --6.571,2 --6.459,2 --6.354,2 --6.244,2 --6.131,2 --6.019,2 --5.911,2 --5.799,2 --5.695,2 --5.584,2 --5.477,2 --5.368,2 --5.261,2 --5.148,2 --5.035,2 --4.929,2 --4.826,2 --4.713,2 --4.606,2 --4.491,2 --4.38,2 --4.27,2 --4.163,2 --4.046,2 --3.935,2 --3.826,2 --3.717,2 --3.61,2 --3.504,2 --3.397,2 --3.289,2 --3.177,2 --3.066,2 --2.953,2 --2.843,2 --2.736,2 --2.631,2 --2.525,2 --2.412,2 --2.304,2 --2.19,2 --2.079,2 --1.971,2 --1.86,2 --1.749,2 --1.638,2 --1.533,2 --1.424,2 --1.319,2 --1.21,2 --1.098,2 --0.988,2 --0.879,2 --0.775,2 --0.666,2 --0.555,2 --0.45,2 --0.345,2 --0.241,2 --0.127,2 --0.014,2 -0.096,2 -0.207,2 -0.32,2 -0.423,2 -0.531,2 -0.641,2 -0.752,2 -0.859,1 -0.97,1 -1.08,1 -1.193,1 -1.301,1 -1.413,1 -1.524,1 -1.636,1 -1.744,1 -1.855,1 -1.96,1 -2.072,1 -2.184,1 -2.297,1 -2.403,1 -2.516,1 -2.621,1 -2.73,1 -2.836,1 -2.945,1 -3.057,1 -3.164,1 -3.275,1 -3.386,1 -3.496,1 -3.611,1 -3.719,1 -3.83,1 -3.936,1 -4.05,1 -4.159,1 -4.267,1 -4.379,1 -4.484,1 -4.59,1 -4.703,1 -4.811,1 -4.918,1 -5.027,1 -5.136,1 -5.241,1 -5.355,1 -5.469,1 -5.581,1 -5.689,1 -5.799,1 -5.908,1 -6.015,1 -6.129,1 -6.233,1 -6.346,1 -6.454,1 -6.565,1 -6.677,1 -6.79,1 -6.901,1 -7.008,1 -7.121,1 -7.234,1 -7.346,1 -7.449,1 -7.559,1 -7.668,1 -7.775,1 -7.886,1 -7.994,1 -8.105,1 -8.21,1 -8.324,1 -8.432,1 -8.545,1 -8.651,1 -8.761,1 -8.868,1 -8.978,1 -9.091,1 -9.199,1 -9.306,1 -9.414,1 -9.527,1 -9.636,1 -9.747,1 -9.853,1 -9.965,1 -10.068,1 -10.176,1 -10.285,1 -10.401,1 -10.508,1 -10.619,1 -10.728,1 -10.837,1 -10.949,1 -11.061,1 -11.171,1 -11.283,1 -11.393,1 -11.501,1 -11.617,1 -11.725,1 -11.835,1 -11.942,1 -12.054,1 -12.159,1 -12.27,1 -12.378,1 -12.481,1 -12.591,1 -12.703,1 -12.807,1 -12.911,1 -13.021,1 -13.131,1 -13.241,1 -13.356,1 -13.463,1 -13.574,1 -13.686,1 -13.795,1 -13.906,1 -14.014,1 -14.123,1 -14.232,1 -14.343,2 -14.452,2 -14.565,2 -14.671,2 -14.782,2 -14.886,2 -14.994,2 -15.108,2 -15.223,2 -15.336,2 -15.446,2 -15.56,2 -15.674,2 -15.781,2 -15.895,2 -16.01,2 -16.12,2 -16.227,2 -16.341,2 -16.455,2 -16.566,2 -16.68,2 -16.789,2 -16.9,2 -17.011,2 -17.122,2 -17.23,2 -17.34,2 -17.445,2 -17.556,2 -17.668,2 -17.774,2 -17.877,2 -17.981,2 -18.085,2 -18.191,2 -18.3,2 -18.412,2 -18.527,2 -18.634,2 -18.74,2 -18.846,2 -18.962,2 -19.077,2 -19.188,2 -19.293,2 -19.402,2 -19.51,2 -19.62,2 -19.725,2 -19.83,2 -19.934,2 -20.04,2 -20.156,2 -20.269,2 -20.381,2 -20.493,2 -20.612,2 -20.723,2 -20.827,2 -20.938,2 -21.043,2 -21.151,2 -21.26,2 -21.374,2 -21.489,2 -21.599,2 -21.71,2 -21.818,2 -21.931,2 -22.037,2 -22.142,2 -22.254,2 -22.366,2 -22.48,2 -22.586,2 -22.695,2 -22.808,2 -22.921,2 -23.037,2 -23.148,2 -23.256,2 -23.365,2 -23.47,2 -23.58,2 -23.69,2 -23.798,2 -23.904,2 -24.008,2 -24.114,2 -24.227,2 -24.34,2 -24.447,2 -24.553,2 -24.662,2 -24.775,2 -24.886,2 -24.995,2 -25.108,2 -25.219,2 -25.329,2 -25.441,2 -25.555,2 -25.665,2 -25.78,2 -25.893,2 -25.998,2 -26.103,2 -26.206,2 -26.317,2 -26.423,2 -26.535,2 -26.641,2 -26.747,2 -26.86,2 -26.964,2 -27.075,2 -27.184,2 -27.288,2 -27.399,2 -27.511,2 -27.615,2 -27.721,2 -27.831,2 -27.938,2 -28.043,2 -28.149,2 -28.259,2 -28.364,2 -28.468,2 -28.58,2 -28.689,2 -28.8,2 -28.91,2 -29.017,2 -29.129,2 -29.238,2 -29.345,2 -29.452,2 -29.565,2 -29.676,2 -29.783,2 -29.887,2 -29.994,2 -30.101,2 -30.208,2 -30.315,2 -30.427,2 -30.538,2 -30.651,2 -30.759,2 -30.87,2 -30.977,2 -31.082,2 -31.191,2 -31.299,2 -31.408,2 -31.521,2 -31.632,2 -31.736,2 -31.845,2 -31.957,2 -32.071,2 -32.182,2 -32.295,2 -32.404,2 -32.51,2 -32.622,2 -32.737,2 -32.846,2 -32.961,2 -33.069,2 -33.181,2 -33.292,2 -33.401,2 -33.514,2 -33.621,2 -33.728,2 -33.837,2 -33.942,2 -34.049,2 -34.152,2 -34.261,2 -34.366,2 -34.479,2 -34.588,2 -34.692,2 -34.797,2 -34.901,2 -35.011,2 -35.118,2 -35.227,2 -35.337,2 -35.443,2 -35.556,2 -35.665,2 -35.774,2 -35.886,2 -35.999,2 -36.108,2 -36.22,2 -36.333,2 -36.446,2 -36.554,2 -36.661,2 -36.774,2 -36.886,2 -36.994,2 -37.108,2 -37.217,2 -37.33,2 -37.437,2 -37.54,2 -37.653,2 -37.761,2 -37.872,2 -37.983,2 -38.094,2 -38.201,2 -38.313,2 -38.421,2 -38.532,2 -38.636,2 -38.749,2 -38.856,2 -38.97,2 -39.08,2 -39.194,2 -39.302,2 -39.411,2 -39.524,2 -39.636,2 -39.749,2 -39.856,2 -39.963,2 -40.07,2 -40.182,2 -40.294,2 -40.407,2 -40.521,2 -40.631,2 -40.735,2 -40.844,2 -40.956,2 -41.071,2 -41.181,2 -41.289,2 -41.398,2 -41.508,2 -41.619,2 -41.732,2 -41.842,2 -41.956,2 -42.07,2 -42.178,2 -42.292,2 -42.406,2 -42.516,2 -42.626,2 -42.736,2 -42.846,2 -42.957,2 -43.067,2 -43.178,2 -43.293,2 -43.407,2 -43.512,2 -43.625,2 -43.739,2 -43.844,2 -43.951,2 -44.059,2 -44.171,2 -44.285,2 -44.397,2 -44.507,2 -44.619,2 -44.729,2 -44.843,2 -44.951,2 -45.058,2 -45.169,2 -45.278,2 -45.387,2 -45.499,2 diff --git a/evidence/failover-each/fe75c6c37e5a/probe.csv b/evidence/failover-each/fe75c6c37e5a/probe.csv deleted file mode 100644 index 531ebbe6..00000000 --- a/evidence/failover-each/fe75c6c37e5a/probe.csv +++ /dev/null @@ -1,647 +0,0 @@ -t_rel_s,status,latency_ms,instance --25.746,201,3.9,fe75c6c37e5a --25.642,201,3.7,ff81f93e8b0d --25.536,201,4.0,fe75c6c37e5a --25.424,201,5.4,fe75c6c37e5a --25.315,201,4.1,fe75c6c37e5a --25.203,201,3.3,ff81f93e8b0d --25.091,201,4.0,ff81f93e8b0d --24.978,201,4.8,fe75c6c37e5a --24.864,201,3.1,fe75c6c37e5a --24.759,201,4.8,fe75c6c37e5a --24.653,201,3.6,fe75c6c37e5a --24.54,201,5.0,fe75c6c37e5a --24.428,201,3.1,fe75c6c37e5a --24.316,201,3.0,fe75c6c37e5a --24.204,201,2.6,fe75c6c37e5a --24.097,201,3.9,fe75c6c37e5a --23.984,201,3.1,fe75c6c37e5a --23.879,201,2.8,fe75c6c37e5a --23.77,201,3.1,fe75c6c37e5a --23.657,201,2.7,fe75c6c37e5a --23.545,201,3.8,fe75c6c37e5a --23.44,201,4.4,fe75c6c37e5a --23.326,201,5.9,fe75c6c37e5a --23.21,201,3.6,ff81f93e8b0d --23.099,201,3.8,ff81f93e8b0d --22.988,201,3.5,ff81f93e8b0d --22.881,201,3.3,ff81f93e8b0d --22.773,201,2.9,fe75c6c37e5a --22.668,201,3.1,fe75c6c37e5a --22.557,201,3.5,fe75c6c37e5a --22.447,201,3.5,fe75c6c37e5a --22.339,201,3.2,fe75c6c37e5a --22.235,201,2.8,ff81f93e8b0d --22.13,201,2.8,ff81f93e8b0d --22.021,201,2.9,ff81f93e8b0d --21.918,201,2.7,fe75c6c37e5a --21.808,201,2.6,fe75c6c37e5a --21.697,201,3.7,fe75c6c37e5a --21.588,201,4.6,fe75c6c37e5a --21.474,201,3.8,fe75c6c37e5a --21.365,201,3.2,fe75c6c37e5a --21.258,201,3.0,fe75c6c37e5a --21.153,201,7.8,fe75c6c37e5a --21.038,201,3.3,fe75c6c37e5a --20.929,201,4.0,fe75c6c37e5a --20.817,201,4.3,fe75c6c37e5a --20.704,201,3.3,fe75c6c37e5a --20.592,201,3.3,fe75c6c37e5a --20.481,201,3.1,ff81f93e8b0d --20.375,201,5.8,fe75c6c37e5a --20.265,201,3.3,fe75c6c37e5a --20.156,201,3.3,fe75c6c37e5a --20.047,201,2.8,fe75c6c37e5a --19.938,201,3.5,fe75c6c37e5a --19.827,201,5.8,fe75c6c37e5a --19.712,201,3.9,fe75c6c37e5a --19.608,201,5.0,fe75c6c37e5a --19.502,201,3.0,ff81f93e8b0d --19.391,201,4.4,fe75c6c37e5a --19.278,201,3.1,fe75c6c37e5a --19.172,201,2.9,fe75c6c37e5a --19.066,201,2.9,ff81f93e8b0d --18.953,201,3.3,ff81f93e8b0d --18.842,201,5.1,fe75c6c37e5a --18.73,201,4.1,fe75c6c37e5a --18.618,201,5.5,fe75c6c37e5a --18.505,201,3.9,fe75c6c37e5a --18.391,201,5.0,ff81f93e8b0d --18.278,201,4.5,ff81f93e8b0d --18.172,201,4.2,ff81f93e8b0d --18.064,201,3.2,fe75c6c37e5a --17.956,201,3.7,fe75c6c37e5a --17.844,201,3.6,fe75c6c37e5a --17.735,201,3.7,fe75c6c37e5a --17.63,201,3.3,fe75c6c37e5a --17.518,201,4.1,fe75c6c37e5a --17.411,201,4.2,fe75c6c37e5a --17.3,201,3.4,fe75c6c37e5a --17.195,201,3.4,ff81f93e8b0d --17.089,201,4.0,ff81f93e8b0d --16.975,201,4.4,ff81f93e8b0d --16.864,201,3.5,ff81f93e8b0d --16.759,201,4.8,ff81f93e8b0d --16.646,201,4.1,ff81f93e8b0d --16.534,201,3.6,ff81f93e8b0d --16.424,201,4.4,ff81f93e8b0d --16.31,201,5.0,fe75c6c37e5a --16.195,201,6.2,fe75c6c37e5a --16.085,201,4.2,fe75c6c37e5a --15.972,201,4.0,ff81f93e8b0d --15.861,201,3.0,ff81f93e8b0d --15.756,201,3.6,fe75c6c37e5a --15.644,201,6.6,ff81f93e8b0d --15.529,201,4.7,ff81f93e8b0d --15.42,201,3.6,ff81f93e8b0d --15.313,201,4.7,fe75c6c37e5a --15.202,201,3.6,ff81f93e8b0d --15.092,201,5.8,ff81f93e8b0d --14.985,201,4.6,ff81f93e8b0d --14.877,201,3.4,ff81f93e8b0d --14.765,201,3.4,ff81f93e8b0d --14.656,201,4.5,ff81f93e8b0d --14.545,201,7.4,ff81f93e8b0d --14.434,201,3.3,ff81f93e8b0d --14.322,201,3.7,fe75c6c37e5a --14.212,201,4.5,fe75c6c37e5a --14.105,201,4.5,ff81f93e8b0d --13.992,201,4.9,fe75c6c37e5a --13.882,201,3.5,fe75c6c37e5a --13.77,201,3.4,fe75c6c37e5a --13.664,201,3.2,ff81f93e8b0d --13.559,201,5.3,ff81f93e8b0d --13.446,201,3.4,ff81f93e8b0d --13.333,201,3.9,fe75c6c37e5a --13.224,201,5.2,fe75c6c37e5a --13.115,201,3.6,ff81f93e8b0d --13.003,201,3.6,ff81f93e8b0d --12.892,201,4.8,fe75c6c37e5a --12.778,201,3.0,fe75c6c37e5a --12.674,201,2.9,ff81f93e8b0d --12.57,201,3.9,ff81f93e8b0d --12.463,201,3.5,fe75c6c37e5a --12.351,201,5.9,ff81f93e8b0d --12.238,201,2.9,ff81f93e8b0d --12.133,201,3.6,fe75c6c37e5a --12.023,201,5.1,ff81f93e8b0d --11.91,201,4.8,ff81f93e8b0d --11.801,201,4.4,ff81f93e8b0d --11.687,201,3.9,ff81f93e8b0d --11.578,201,3.0,ff81f93e8b0d --11.467,201,2.8,ff81f93e8b0d --11.358,201,4.5,ff81f93e8b0d --11.247,201,3.5,ff81f93e8b0d --11.139,201,4.9,ff81f93e8b0d --11.033,201,2.8,fe75c6c37e5a --10.925,201,3.9,fe75c6c37e5a --10.812,201,4.6,ff81f93e8b0d --10.705,201,2.8,fe75c6c37e5a --10.598,201,4.4,fe75c6c37e5a --10.488,201,3.1,fe75c6c37e5a --10.383,201,5.0,fe75c6c37e5a --10.273,201,4.1,fe75c6c37e5a --10.166,201,3.7,ff81f93e8b0d --10.057,201,4.6,ff81f93e8b0d --9.945,201,3.5,ff81f93e8b0d --9.841,201,4.4,ff81f93e8b0d --9.735,201,4.7,fe75c6c37e5a --9.621,201,2.6,ff81f93e8b0d --9.51,201,2.6,ff81f93e8b0d --9.401,201,3.6,ff81f93e8b0d --9.287,201,4.5,ff81f93e8b0d --9.172,201,3.9,fe75c6c37e5a --9.065,201,4.7,ff81f93e8b0d --8.951,201,3.5,fe75c6c37e5a --8.84,201,3.3,fe75c6c37e5a --8.727,201,3.0,fe75c6c37e5a --8.618,201,3.1,fe75c6c37e5a --8.506,201,3.7,fe75c6c37e5a --8.394,201,3.6,fe75c6c37e5a --8.287,201,3.3,fe75c6c37e5a --8.182,201,3.9,fe75c6c37e5a --8.07,201,4.0,fe75c6c37e5a --7.961,201,3.1,fe75c6c37e5a --7.857,201,4.2,ff81f93e8b0d --7.745,201,3.6,ff81f93e8b0d --7.633,201,3.1,fe75c6c37e5a --7.52,201,4.0,fe75c6c37e5a --7.413,201,4.0,ff81f93e8b0d --7.304,201,3.0,ff81f93e8b0d --7.2,201,4.4,ff81f93e8b0d --7.095,201,4.2,fe75c6c37e5a --6.983,201,3.4,fe75c6c37e5a --6.876,201,2.9,fe75c6c37e5a --6.763,201,2.8,fe75c6c37e5a --6.652,201,3.6,ff81f93e8b0d --6.542,201,3.4,ff81f93e8b0d --6.432,201,4.6,ff81f93e8b0d --6.32,201,3.2,ff81f93e8b0d --6.211,201,2.9,ff81f93e8b0d --6.104,201,4.6,fe75c6c37e5a --5.993,201,3.8,ff81f93e8b0d --5.889,201,3.0,fe75c6c37e5a --5.778,201,3.6,fe75c6c37e5a --5.673,201,4.2,ff81f93e8b0d --5.565,201,4.5,ff81f93e8b0d --5.459,201,4.5,ff81f93e8b0d --5.345,201,3.5,ff81f93e8b0d --5.239,201,3.1,ff81f93e8b0d --5.128,201,3.3,ff81f93e8b0d --5.018,201,3.0,ff81f93e8b0d --4.913,201,4.3,ff81f93e8b0d --4.805,201,3.5,fe75c6c37e5a --4.692,201,3.4,ff81f93e8b0d --4.578,201,2.8,ff81f93e8b0d --4.465,201,3.9,ff81f93e8b0d --4.352,201,4.0,fe75c6c37e5a --4.242,201,4.4,fe75c6c37e5a --4.132,201,8.4,fe75c6c37e5a --4.016,201,4.6,fe75c6c37e5a --3.903,201,4.2,fe75c6c37e5a --3.789,201,3.2,ff81f93e8b0d --3.684,201,4.0,fe75c6c37e5a --3.579,201,5.5,fe75c6c37e5a --3.47,201,4.8,fe75c6c37e5a --3.363,201,4.6,fe75c6c37e5a --3.252,201,3.9,fe75c6c37e5a --3.14,201,3.2,fe75c6c37e5a --3.031,201,4.0,fe75c6c37e5a --2.922,201,3.6,fe75c6c37e5a --2.817,201,3.1,ff81f93e8b0d --2.708,201,3.3,ff81f93e8b0d --2.598,201,3.1,ff81f93e8b0d --2.486,201,4.4,ff81f93e8b0d --2.377,201,4.9,ff81f93e8b0d --2.271,201,3.6,ff81f93e8b0d --2.162,201,6.7,ff81f93e8b0d --2.054,201,3.8,fe75c6c37e5a --1.942,201,6.2,ff81f93e8b0d --1.829,201,3.7,ff81f93e8b0d --1.723,201,4.2,ff81f93e8b0d --1.614,201,3.5,fe75c6c37e5a --1.505,201,5.7,fe75c6c37e5a --1.398,201,2.8,fe75c6c37e5a --1.293,201,5.8,fe75c6c37e5a --1.177,201,5.6,fe75c6c37e5a --1.068,201,3.5,fe75c6c37e5a --0.956,201,3.5,fe75c6c37e5a --0.843,201,5.6,fe75c6c37e5a --0.729,201,4.6,ff81f93e8b0d --0.618,201,4.8,ff81f93e8b0d --0.504,201,4.4,ff81f93e8b0d --0.39,201,5.0,fe75c6c37e5a --0.284,201,3.6,ff81f93e8b0d --0.17,201,3.3,fe75c6c37e5a --0.066,201,4.6,ff81f93e8b0d -0.048,201,4.6,ff81f93e8b0d -0.155,201,3.4,ff81f93e8b0d -0.26,201,2.9,ff81f93e8b0d -0.37,201,3.5,ff81f93e8b0d -0.481,201,3.7,ff81f93e8b0d -0.589,201,5.1,ff81f93e8b0d -0.702,201,4.7,ff81f93e8b0d -0.807,201,5.9,ff81f93e8b0d -0.923,201,9.2,ff81f93e8b0d -1.039,201,5.3,ff81f93e8b0d -1.154,201,3.0,ff81f93e8b0d -1.262,201,4.2,ff81f93e8b0d -1.377,201,4.8,ff81f93e8b0d -1.491,201,6.2,ff81f93e8b0d -1.6,201,3.2,ff81f93e8b0d -1.709,201,5.1,ff81f93e8b0d -1.816,201,3.3,ff81f93e8b0d -1.928,201,3.5,ff81f93e8b0d -2.036,201,3.4,ff81f93e8b0d -2.145,201,4.1,ff81f93e8b0d -2.259,201,3.9,ff81f93e8b0d -2.372,201,5.5,ff81f93e8b0d -2.486,201,4.0,ff81f93e8b0d -2.592,201,5.0,ff81f93e8b0d -2.699,201,3.2,ff81f93e8b0d -2.812,201,3.1,ff81f93e8b0d -2.922,201,5.1,ff81f93e8b0d -3.034,201,4.3,ff81f93e8b0d -3.142,201,4.1,ff81f93e8b0d -3.254,201,3.2,ff81f93e8b0d -3.361,201,3.8,ff81f93e8b0d -3.471,201,4.2,ff81f93e8b0d -3.584,201,3.3,ff81f93e8b0d -3.692,201,3.6,ff81f93e8b0d -3.798,201,4.2,ff81f93e8b0d -3.911,201,3.5,ff81f93e8b0d -4.022,201,6.0,ff81f93e8b0d -4.129,201,3.7,ff81f93e8b0d -4.241,201,4.9,ff81f93e8b0d -4.356,201,5.4,ff81f93e8b0d -4.471,201,3.6,ff81f93e8b0d -4.581,201,3.9,ff81f93e8b0d -4.687,201,5.7,ff81f93e8b0d -4.797,201,5.9,ff81f93e8b0d -4.913,201,7.6,ff81f93e8b0d -5.027,201,4.0,ff81f93e8b0d -5.141,201,4.4,ff81f93e8b0d -5.254,201,3.8,ff81f93e8b0d -5.366,201,4.1,ff81f93e8b0d -5.475,201,4.1,ff81f93e8b0d -5.587,201,3.0,ff81f93e8b0d -5.695,201,2.6,ff81f93e8b0d -5.799,201,4.1,ff81f93e8b0d -5.911,201,4.3,ff81f93e8b0d -6.018,201,4.5,ff81f93e8b0d -6.123,201,3.9,ff81f93e8b0d -6.237,201,5.0,ff81f93e8b0d -6.35,201,2.4,ff81f93e8b0d -6.462,201,2.4,ff81f93e8b0d -6.568,201,4.3,ff81f93e8b0d -6.68,201,5.2,ff81f93e8b0d -6.793,201,4.5,ff81f93e8b0d -6.899,201,4.1,ff81f93e8b0d -7.008,201,5.6,ff81f93e8b0d -7.123,201,3.1,ff81f93e8b0d -7.229,201,4.4,ff81f93e8b0d -7.336,201,4.9,ff81f93e8b0d -7.446,201,4.0,ff81f93e8b0d -7.559,201,4.2,ff81f93e8b0d -7.666,201,5.1,ff81f93e8b0d -7.782,201,3.6,ff81f93e8b0d -7.893,201,4.0,ff81f93e8b0d -8.004,201,3.2,ff81f93e8b0d -8.11,201,3.9,ff81f93e8b0d -8.219,201,2.5,ff81f93e8b0d -8.327,201,3.3,ff81f93e8b0d -8.43,201,5.6,ff81f93e8b0d -8.543,201,6.1,ff81f93e8b0d -8.658,201,3.6,ff81f93e8b0d -8.762,201,3.9,ff81f93e8b0d -8.876,201,4.5,ff81f93e8b0d -8.99,201,6.7,ff81f93e8b0d -9.107,201,2.7,ff81f93e8b0d -9.219,201,5.0,ff81f93e8b0d -9.334,201,4.0,ff81f93e8b0d -9.445,201,5.6,ff81f93e8b0d -9.553,201,3.5,ff81f93e8b0d -9.66,201,4.1,ff81f93e8b0d -9.768,201,3.9,ff81f93e8b0d -9.873,201,3.5,ff81f93e8b0d -9.982,201,2.7,ff81f93e8b0d -10.093,201,4.2,ff81f93e8b0d -10.202,201,4.4,ff81f93e8b0d -10.315,201,4.8,ff81f93e8b0d -10.423,201,3.4,ff81f93e8b0d -10.532,201,3.7,ff81f93e8b0d -10.639,201,3.8,ff81f93e8b0d -10.75,201,2.9,ff81f93e8b0d -10.855,201,3.9,ff81f93e8b0d -10.966,201,4.3,ff81f93e8b0d -11.081,201,3.2,ff81f93e8b0d -11.192,201,5.1,ff81f93e8b0d -11.303,201,3.2,ff81f93e8b0d -11.409,201,5.2,ff81f93e8b0d -11.514,201,4.8,ff81f93e8b0d -11.624,201,3.6,ff81f93e8b0d -11.734,201,4.6,ff81f93e8b0d -11.848,201,3.0,ff81f93e8b0d -11.954,201,3.4,ff81f93e8b0d -12.065,201,5.1,ff81f93e8b0d -12.175,201,4.6,ff81f93e8b0d -12.287,201,5.6,ff81f93e8b0d -12.403,201,2.8,ff81f93e8b0d -12.513,201,4.6,ff81f93e8b0d -12.628,201,2.9,ff81f93e8b0d -12.734,201,3.1,ff81f93e8b0d -12.839,201,6.0,ff81f93e8b0d -12.949,201,4.5,ff81f93e8b0d -13.057,201,4.5,ff81f93e8b0d -13.169,201,4.6,ff81f93e8b0d -13.281,201,4.2,ff81f93e8b0d -13.389,201,3.8,ff81f93e8b0d -13.498,201,3.8,ff81f93e8b0d -13.604,201,3.3,ff81f93e8b0d -13.71,201,6.1,ff81f93e8b0d -13.823,201,3.4,ff81f93e8b0d -13.929,201,3.0,ff81f93e8b0d -14.042,201,6.6,ff81f93e8b0d -14.159,201,3.7,ff81f93e8b0d -14.264,201,3.6,ff81f93e8b0d -14.37,201,3.5,ff81f93e8b0d -14.481,201,2.8,ff81f93e8b0d -14.593,201,4.2,ff81f93e8b0d -14.701,201,4.6,ff81f93e8b0d -14.814,201,5.1,ff81f93e8b0d -14.92,201,5.1,ff81f93e8b0d -15.027,201,5.8,ff81f93e8b0d -15.136,201,3.8,fe75c6c37e5a -15.245,201,7.4,ff81f93e8b0d -15.358,201,3.7,ff81f93e8b0d -15.467,201,5.1,ff81f93e8b0d -15.574,201,5.2,fe75c6c37e5a -15.683,201,3.3,fe75c6c37e5a -15.795,201,4.5,fe75c6c37e5a -15.905,201,4.0,fe75c6c37e5a -16.015,201,3.5,fe75c6c37e5a -16.12,201,4.2,fe75c6c37e5a -16.227,201,5.7,fe75c6c37e5a -16.342,201,5.6,fe75c6c37e5a -16.456,201,3.6,fe75c6c37e5a -16.567,201,5.1,ff81f93e8b0d -16.682,201,4.2,ff81f93e8b0d -16.794,201,3.9,ff81f93e8b0d -16.898,201,4.9,ff81f93e8b0d -17.011,201,5.5,ff81f93e8b0d -17.123,201,2.8,ff81f93e8b0d -17.236,201,3.8,ff81f93e8b0d -17.349,201,4.2,fe75c6c37e5a -17.459,201,4.0,fe75c6c37e5a -17.571,201,4.5,fe75c6c37e5a -17.68,201,5.2,fe75c6c37e5a -17.795,201,5.0,fe75c6c37e5a -17.902,201,3.5,fe75c6c37e5a -18.011,201,3.5,fe75c6c37e5a -18.124,201,5.0,fe75c6c37e5a -18.235,201,4.8,fe75c6c37e5a -18.343,201,4.7,fe75c6c37e5a -18.456,201,6.4,ff81f93e8b0d -18.57,201,8.4,ff81f93e8b0d -18.686,201,5.4,fe75c6c37e5a -18.798,201,14.1,fe75c6c37e5a -18.918,201,12.1,ff81f93e8b0d -19.038,201,6.1,fe75c6c37e5a -19.148,201,5.5,fe75c6c37e5a -19.258,201,6.2,fe75c6c37e5a -19.364,201,7.6,ff81f93e8b0d -19.477,201,5.7,ff81f93e8b0d -19.586,201,4.6,ff81f93e8b0d -19.695,201,4.3,ff81f93e8b0d -19.808,201,6.6,fe75c6c37e5a -19.916,201,7.9,fe75c6c37e5a -20.025,201,6.3,ff81f93e8b0d -20.132,201,7.5,ff81f93e8b0d -20.243,201,5.6,ff81f93e8b0d -20.355,201,6.9,ff81f93e8b0d -20.471,201,33.0,ff81f93e8b0d -20.608,201,11.1,ff81f93e8b0d -20.729,201,5.7,fe75c6c37e5a -20.837,201,5.5,fe75c6c37e5a -20.949,201,4.3,fe75c6c37e5a -21.06,201,7.0,fe75c6c37e5a -21.168,201,7.4,fe75c6c37e5a -21.286,201,6.2,fe75c6c37e5a -21.392,201,5.2,fe75c6c37e5a -21.502,201,3.8,fe75c6c37e5a -21.606,201,5.5,ff81f93e8b0d -21.72,201,3.6,fe75c6c37e5a -21.832,201,4.6,fe75c6c37e5a -21.938,201,8.5,fe75c6c37e5a -22.047,201,3.5,fe75c6c37e5a -22.155,201,5.7,ff81f93e8b0d -22.269,201,4.8,fe75c6c37e5a -22.377,201,4.3,ff81f93e8b0d -22.491,201,3.2,fe75c6c37e5a -22.597,201,4.0,ff81f93e8b0d -22.71,201,4.2,fe75c6c37e5a -22.821,201,5.7,fe75c6c37e5a -22.934,201,10.9,fe75c6c37e5a -23.046,201,10.4,ff81f93e8b0d -23.166,201,6.9,ff81f93e8b0d -23.278,201,3.6,fe75c6c37e5a -23.389,201,6.7,fe75c6c37e5a -23.505,201,5.5,fe75c6c37e5a -23.613,201,4.9,fe75c6c37e5a -23.72,201,10.4,fe75c6c37e5a -23.832,201,3.7,fe75c6c37e5a -23.946,201,5.6,ff81f93e8b0d -24.06,201,3.5,ff81f93e8b0d -24.172,201,5.9,ff81f93e8b0d -24.282,201,3.8,ff81f93e8b0d -24.391,201,4.4,ff81f93e8b0d -24.497,201,4.2,ff81f93e8b0d -24.61,201,5.1,ff81f93e8b0d -24.722,201,6.9,ff81f93e8b0d -24.838,201,7.2,fe75c6c37e5a -24.947,201,5.4,fe75c6c37e5a -25.06,201,4.3,fe75c6c37e5a -25.172,201,6.2,fe75c6c37e5a -25.285,201,3.8,fe75c6c37e5a -25.39,201,6.5,fe75c6c37e5a -25.502,201,5.1,fe75c6c37e5a -25.614,201,3.7,fe75c6c37e5a -25.728,201,4.7,ff81f93e8b0d -25.833,201,5.0,fe75c6c37e5a -25.943,201,4.8,fe75c6c37e5a -26.049,201,5.0,fe75c6c37e5a -26.162,201,3.7,fe75c6c37e5a -26.269,201,3.7,fe75c6c37e5a -26.377,201,3.9,fe75c6c37e5a -26.488,201,6.4,fe75c6c37e5a -26.595,201,4.6,ff81f93e8b0d -26.71,201,7.1,fe75c6c37e5a -26.825,201,6.5,ff81f93e8b0d -26.934,201,4.0,fe75c6c37e5a -27.041,201,4.7,fe75c6c37e5a -27.152,201,5.7,fe75c6c37e5a -27.259,201,4.6,fe75c6c37e5a -27.371,201,3.3,fe75c6c37e5a -27.478,201,4.0,fe75c6c37e5a -27.584,201,4.8,ff81f93e8b0d -27.691,201,5.0,ff81f93e8b0d -27.796,201,5.0,ff81f93e8b0d -27.903,201,6.3,fe75c6c37e5a -28.016,201,4.7,ff81f93e8b0d -28.122,201,3.9,fe75c6c37e5a -28.234,201,3.8,fe75c6c37e5a -28.348,201,4.8,ff81f93e8b0d -28.462,201,5.1,ff81f93e8b0d -28.575,201,5.4,ff81f93e8b0d -28.69,201,3.6,fe75c6c37e5a -28.803,201,3.8,fe75c6c37e5a -28.915,201,2.7,fe75c6c37e5a -29.022,201,2.7,fe75c6c37e5a -29.132,201,3.1,fe75c6c37e5a -29.24,201,4.1,fe75c6c37e5a -29.345,201,5.4,ff81f93e8b0d -29.452,201,4.4,ff81f93e8b0d -29.564,201,4.4,ff81f93e8b0d -29.677,201,4.7,fe75c6c37e5a -29.783,201,3.1,ff81f93e8b0d -29.896,201,3.0,ff81f93e8b0d -30.008,201,3.7,fe75c6c37e5a -30.122,201,3.9,fe75c6c37e5a -30.232,201,4.8,fe75c6c37e5a -30.339,201,4.1,fe75c6c37e5a -30.452,201,4.5,fe75c6c37e5a -30.563,201,3.8,fe75c6c37e5a -30.67,201,2.8,fe75c6c37e5a -30.777,201,5.9,fe75c6c37e5a -30.886,201,5.9,fe75c6c37e5a -30.994,201,5.0,ff81f93e8b0d -31.102,201,3.3,ff81f93e8b0d -31.21,201,4.0,ff81f93e8b0d -31.322,201,6.4,ff81f93e8b0d -31.433,201,31.4,fe75c6c37e5a -31.566,201,4.3,ff81f93e8b0d -31.679,201,8.0,fe75c6c37e5a -31.798,201,3.8,ff81f93e8b0d -31.909,201,5.1,ff81f93e8b0d -32.019,201,3.6,ff81f93e8b0d -32.126,201,5.0,fe75c6c37e5a -32.24,201,6.7,ff81f93e8b0d -32.356,201,5.7,fe75c6c37e5a -32.469,201,6.4,fe75c6c37e5a -32.577,201,4.1,ff81f93e8b0d -32.684,201,3.5,ff81f93e8b0d -32.789,201,7.0,ff81f93e8b0d -32.899,201,5.3,ff81f93e8b0d -33.012,201,5.0,ff81f93e8b0d -33.126,201,3.9,ff81f93e8b0d -33.237,201,8.1,fe75c6c37e5a -33.349,201,4.8,fe75c6c37e5a -33.464,201,3.7,fe75c6c37e5a -33.571,201,4.4,fe75c6c37e5a -33.68,201,4.1,fe75c6c37e5a -33.791,201,5.0,fe75c6c37e5a -33.898,201,4.9,fe75c6c37e5a -34.013,201,4.1,fe75c6c37e5a -34.127,201,4.2,fe75c6c37e5a -34.234,201,4.9,fe75c6c37e5a -34.343,201,4.0,fe75c6c37e5a -34.454,201,4.9,fe75c6c37e5a -34.566,201,4.9,fe75c6c37e5a -34.679,201,3.6,ff81f93e8b0d -34.787,201,5.5,ff81f93e8b0d -34.896,201,5.1,fe75c6c37e5a -35.011,201,5.3,ff81f93e8b0d -35.122,201,6.6,ff81f93e8b0d -35.235,201,3.9,ff81f93e8b0d -35.349,201,3.8,fe75c6c37e5a -35.462,201,3.0,fe75c6c37e5a -35.575,201,3.8,fe75c6c37e5a -35.689,201,10.8,ff81f93e8b0d -35.807,201,5.1,fe75c6c37e5a -35.912,201,4.2,ff81f93e8b0d -36.02,201,8.2,ff81f93e8b0d -36.133,201,4.1,ff81f93e8b0d -36.247,201,4.1,fe75c6c37e5a -36.355,201,6.2,ff81f93e8b0d -36.465,201,3.0,ff81f93e8b0d -36.574,201,9.7,fe75c6c37e5a -36.688,201,8.0,fe75c6c37e5a -36.806,201,7.5,ff81f93e8b0d -36.92,201,6.5,ff81f93e8b0d -37.03,201,5.9,ff81f93e8b0d -37.146,201,4.6,ff81f93e8b0d -37.253,201,5.6,ff81f93e8b0d -37.366,201,3.0,ff81f93e8b0d -37.472,201,5.2,ff81f93e8b0d -37.581,201,5.0,ff81f93e8b0d -37.695,201,5.6,ff81f93e8b0d -37.802,201,6.3,ff81f93e8b0d -37.913,201,4.5,ff81f93e8b0d -38.026,201,3.6,ff81f93e8b0d -38.132,201,4.5,ff81f93e8b0d -38.238,201,4.4,ff81f93e8b0d -38.343,201,3.2,fe75c6c37e5a -38.453,201,4.6,fe75c6c37e5a -38.561,201,4.8,fe75c6c37e5a -38.667,201,4.7,fe75c6c37e5a -38.78,201,4.3,fe75c6c37e5a -38.885,201,4.6,ff81f93e8b0d -38.997,201,4.4,fe75c6c37e5a -39.108,201,4.0,fe75c6c37e5a -39.219,201,3.8,fe75c6c37e5a -39.326,201,3.8,ff81f93e8b0d -39.436,201,4.5,fe75c6c37e5a -39.545,201,4.4,ff81f93e8b0d -39.653,201,3.7,ff81f93e8b0d -39.758,201,4.2,ff81f93e8b0d -39.872,201,6.1,ff81f93e8b0d -39.988,201,5.5,fe75c6c37e5a -40.096,201,5.1,ff81f93e8b0d -40.21,201,7.3,fe75c6c37e5a -40.32,201,5.7,fe75c6c37e5a -40.428,201,4.6,fe75c6c37e5a -40.536,201,4.1,ff81f93e8b0d -40.646,201,4.2,ff81f93e8b0d -40.756,201,6.5,fe75c6c37e5a -40.869,201,4.4,fe75c6c37e5a -40.98,201,4.4,fe75c6c37e5a -41.093,201,4.3,fe75c6c37e5a -41.204,201,3.3,fe75c6c37e5a -41.309,201,6.1,fe75c6c37e5a -41.416,201,4.9,fe75c6c37e5a -41.529,201,5.6,fe75c6c37e5a -41.641,201,4.8,fe75c6c37e5a -41.747,201,5.8,fe75c6c37e5a -41.863,201,18.6,fe75c6c37e5a -41.992,201,5.2,fe75c6c37e5a -42.104,201,6.8,fe75c6c37e5a -42.221,201,5.9,ff81f93e8b0d -42.331,201,5.3,ff81f93e8b0d -42.444,201,4.3,ff81f93e8b0d -42.551,201,4.8,ff81f93e8b0d -42.664,201,4.8,ff81f93e8b0d -42.769,201,4.3,ff81f93e8b0d -42.876,201,4.0,fe75c6c37e5a -42.988,201,7.2,ff81f93e8b0d -43.098,201,3.6,fe75c6c37e5a -43.211,201,4.7,ff81f93e8b0d -43.317,201,4.7,ff81f93e8b0d -43.43,201,4.6,ff81f93e8b0d -43.542,201,3.9,ff81f93e8b0d -43.655,201,6.3,fe75c6c37e5a -43.769,201,4.7,fe75c6c37e5a -43.879,201,14.6,ff81f93e8b0d -44.001,201,4.4,ff81f93e8b0d -44.112,201,4.8,ff81f93e8b0d -44.225,201,4.6,ff81f93e8b0d -44.34,201,6.7,fe75c6c37e5a -44.451,201,6.0,fe75c6c37e5a -44.561,201,5.1,fe75c6c37e5a -44.672,201,6.5,fe75c6c37e5a -44.782,201,3.5,fe75c6c37e5a -44.893,201,3.4,fe75c6c37e5a -45.006,201,6.0,fe75c6c37e5a -45.112,201,6.0,ff81f93e8b0d -45.226,201,6.8,fe75c6c37e5a -45.339,201,11.3,ff81f93e8b0d -45.459,201,5.7,fe75c6c37e5a diff --git a/evidence/failover-each/ff81f93e8b0d/action.txt b/evidence/failover-each/ff81f93e8b0d/action.txt deleted file mode 100644 index e5e33782..00000000 --- a/evidence/failover-each/ff81f93e8b0d/action.txt +++ /dev/null @@ -1,14 +0,0 @@ -ff81f93e8b0d api Exited (0) Less than a second ago -fe75c6c37e5a api Up About a minute (healthy) -02:48:39 database bytes,messages: 9672371,2759 -mtusi2026lab3-api-2 cpu=12.79% mem=125.6MiB / 512MiB -mtusi2026lab3-lb-1 cpu=3.32% mem=19.51MiB / 128MiB -mtusi2026lab3-postgres-1 cpu=4.32% mem=54.65MiB / 1GiB -mtusi2026lab3-valkey-1 cpu=1.20% mem=13.09MiB / 128MiB - Container mtusi2026lab3-api-2 Running - Container mtusi2026lab3-api-1 Starting - Container mtusi2026lab3-api-1 Started - Container mtusi2026lab3-api-1 Waiting - Container mtusi2026lab3-api-2 Waiting - Container mtusi2026lab3-api-1 Healthy - Container mtusi2026lab3-api-2 Healthy diff --git a/evidence/failover-each/ff81f93e8b0d/k6/k6.txt b/evidence/failover-each/ff81f93e8b0d/k6/k6.txt deleted file mode 100644 index 5d53c98a..00000000 --- a/evidence/failover-each/ff81f93e8b0d/k6/k6.txt +++ /dev/null @@ -1,2 +0,0 @@ -time="2026-10-30T02:48:11Z" level=info msg=LOAD-START source=console -measure: 6001 requests, failed ratio 0 diff --git a/evidence/failover-each/ff81f93e8b0d/k6/summary.json b/evidence/failover-each/ff81f93e8b0d/k6/summary.json deleted file mode 100644 index c813e862..00000000 --- a/evidence/failover-each/ff81f93e8b0d/k6/summary.json +++ /dev/null @@ -1,309 +0,0 @@ -{ - "root_group": { - "name": "", - "path": "", - "id": "d41d8cd98f00b204e9800998ecf8427e", - "groups": [], - "checks": [ - { - "name": "message 201", - "path": "::message 201", - "id": "6445aff337e1ade0bc10402fbac79882", - "passes": 5647, - "fails": 0 - }, - { - "id": "fe27b3b8edd292c7f2170a6357b087d0", - "passes": 1355, - "fails": 0, - "name": "history 200", - "path": "::history 200" - } - ] - }, - "options": { - "summaryTimeUnit": "", - "noColor": false, - "summaryTrendStats": [ - "avg", - "min", - "med", - "p(90)", - "p(95)", - "p(99)", - "max" - ] - }, - "state": { - "isStdOutTTY": false, - "isStdErrTTY": false, - "testRunDurationMs": 70525.755241 - }, - "metrics": { - "http_req_duration{expected_response:true}": { - "contains": "time", - "values": { - "med": 1.566875, - "p(90)": 3.0046625000000007, - "p(95)": 3.926481399999998, - "p(99)": 9.87922391999999, - "max": 31.835458, - "avg": 1.9633167280204142, - "min": 0.883916 - }, - "type": "trend" - }, - "http_req_waiting": { - "values": { - "p(99)": 9.739183839999988, - "max": 31.802583, - "avg": 1.921257879608627, - "min": 0.871708, - "med": 1.528979, - "p(90)": 2.9510836, - "p(95)": 3.8628975499999996 - }, - "type": "trend", - "contains": "time" - }, - "http_reqs": { - "type": "counter", - "contains": "default", - "values": { - "count": 7052, - "rate": 99.9918395187966 - } - }, - "http_req_receiving": { - "values": { - "med": 0.023791, - "p(90)": 0.042791, - "p(95)": 0.05251479999999996, - "p(99)": 0.08951307999999995, - "max": 1.60625, - "avg": 0.027876955757232046, - "min": 0.003375 - }, - "type": "trend", - "contains": "time" - }, - "http_req_connecting": { - "type": "trend", - "contains": "time", - "values": { - "avg": 0.0004884490924560408, - "min": 0, - "med": 0, - "p(90)": 0, - "p(95)": 0, - "p(99)": 0, - "max": 0.33575 - } - }, - "http_req_failed": { - "type": "rate", - "contains": "default", - "values": { - "rate": 0, - "passes": 0, - "fails": 7052 - } - }, - "http_req_tls_handshaking": { - "type": "trend", - "contains": "time", - "values": { - "avg": 0, - "min": 0, - "med": 0, - "p(90)": 0, - "p(95)": 0, - "p(99)": 0, - "max": 0 - } - }, - "data_sent": { - "type": "counter", - "contains": "data", - "values": { - "count": 2057256, - "rate": 29170.279608775018 - } - }, - "checks": { - "contains": "default", - "values": { - "rate": 1, - "passes": 7002, - "fails": 0 - }, - "type": "rate" - }, - "http_req_sending": { - "type": "trend", - "contains": "time", - "values": { - "p(90)": 0.018375, - "p(95)": 0.023042, - "p(99)": 0.05424916, - "max": 1.320584, - "avg": 0.01418189265456617, - "min": 0.003084, - "med": 0.011792 - } - }, - "http_reqs{scenario:measure}": { - "type": "counter", - "contains": "default", - "values": { - "count": 6001, - "rate": 85.08948226776778 - } - }, - "dropped_iterations{scenario:measure}": { - "type": "counter", - "contains": "default", - "values": { - "count": 0, - "rate": 0 - } - }, - "http_req_duration{scenario:measure}": { - "type": "trend", - "contains": "time", - "values": { - "p(90)": 3.067875, - "p(95)": 3.922292, - "p(99)": 7.314708, - "max": 31.835458, - "avg": 1.9610305627395372, - "min": 0.883916, - "med": 1.595209 - } - }, - "http_req_blocked": { - "type": "trend", - "contains": "time", - "values": { - "p(90)": 0.003375, - "p(95)": 0.004125, - "p(99)": 0.006292, - "max": 0.932542, - "avg": 0.00281850311968237, - "min": 0.000708, - "med": 0.001875 - } - }, - "data_received": { - "type": "counter", - "contains": "data", - "values": { - "count": 16698312, - "rate": 236768.99230555806 - } - }, - "vus": { - "type": "gauge", - "contains": "default", - "values": { - "value": 1, - "min": 0, - "max": 1 - } - }, - "http_req_duration": { - "type": "trend", - "contains": "time", - "values": { - "med": 1.566875, - "p(90)": 3.0046625000000007, - "p(95)": 3.926481399999998, - "p(99)": 9.87922391999999, - "max": 31.835458, - "avg": 1.9633167280204142, - "min": 0.883916 - } - }, - "dropped_iterations": { - "type": "counter", - "contains": "default", - "values": { - "rate": 0, - "count": 0 - } - }, - "http_req_failed{scenario:measure}": { - "type": "rate", - "contains": "default", - "values": { - "rate": 0, - "passes": 0, - "fails": 6001 - } - }, - "iterations{scenario:measure}": { - "type": "counter", - "contains": "default", - "values": { - "count": 6001, - "rate": 85.08948226776778 - } - }, - "iterations": { - "type": "counter", - "contains": "default", - "values": { - "rate": 99.28287866004166, - "count": 7002 - } - }, - "http_req_duration{scenario:measure,name:get_history}": { - "type": "trend", - "contains": "time", - "values": { - "max": 19.542875, - "avg": 2.039735076521738, - "min": 1.055333, - "med": 1.6431455000000001, - "p(90)": 3.2530464, - "p(95)": 3.958685449999999, - "p(99)": 7.853225919999997 - } - }, - "vus_max": { - "contains": "default", - "values": { - "max": 20, - "value": 20, - "min": 20 - }, - "type": "gauge" - }, - "http_req_duration{scenario:measure,name:post_message}": { - "type": "trend", - "contains": "time", - "values": { - "max": 31.835458, - "avg": 1.942372514739224, - "min": 0.883916, - "med": 1.582625, - "p(90)": 3.030458, - "p(95)": 3.9060624999999978, - "p(99)": 7.141624999999999 - } - }, - "iteration_duration": { - "contains": "time", - "values": { - "med": 1.6496255, - "p(90)": 3.0859256000000004, - "p(95)": 3.944686049999999, - "p(99)": 7.210350089999998, - "max": 31.969333, - "avg": 1.9984389771493831, - "min": 0.9315 - }, - "type": "trend" - } - } -} \ No newline at end of file diff --git a/evidence/failover-each/ff81f93e8b0d/lb_up.csv b/evidence/failover-each/ff81f93e8b0d/lb_up.csv deleted file mode 100644 index cdbf7e50..00000000 --- a/evidence/failover-each/ff81f93e8b0d/lb_up.csv +++ /dev/null @@ -1,655 +0,0 @@ -t_rel_s,up --25.788,2 --25.681,2 --25.575,2 --25.464,2 --25.351,2 --25.244,2 --25.139,2 --25.026,2 --24.911,2 --24.8,2 --24.687,2 --24.584,2 --24.47,2 --24.363,2 --24.251,2 --24.145,2 --24.036,2 --23.924,2 --23.811,2 --23.697,2 --23.587,2 --23.482,2 --23.37,2 --23.258,2 --23.146,2 --23.04,2 --22.928,2 --22.818,2 --22.712,2 --22.604,2 --22.495,2 --22.386,2 --22.283,2 --22.174,2 --22.063,2 --21.95,2 --21.84,2 --21.727,2 --21.619,2 --21.507,2 --21.394,2 --21.279,2 --21.166,2 --21.055,2 --20.942,2 --20.831,2 --20.719,2 --20.61,2 --20.505,2 --20.401,2 --20.291,2 --20.18,2 --20.069,2 --19.956,2 --19.846,2 --19.735,2 --19.626,2 --19.52,2 --19.408,2 --19.303,2 --19.195,2 --19.084,2 --18.98,2 --18.877,2 --18.767,2 --18.657,2 --18.544,2 --18.438,2 --18.325,2 --18.212,2 --18.107,2 --18.002,2 --17.891,2 --17.784,2 --17.67,2 --17.564,2 --17.45,2 --17.341,2 --17.234,2 --17.126,2 --17.013,2 --16.908,2 --16.794,2 --16.688,2 --16.58,2 --16.475,2 --16.368,2 --16.265,2 --16.16,2 --16.047,2 --15.942,2 --15.83,2 --15.721,2 --15.612,2 --15.5,2 --15.389,2 --15.282,2 --15.167,2 --15.055,2 --14.946,2 --14.837,2 --14.728,2 --14.618,2 --14.508,2 --14.398,2 --14.287,2 --14.176,2 --14.067,2 --13.956,2 --13.851,2 --13.738,2 --13.625,2 --13.514,2 --13.405,2 --13.293,2 --13.186,2 --13.075,2 --12.964,2 --12.858,2 --12.748,2 --12.641,2 --12.528,2 --12.415,2 --12.303,2 --12.199,2 --12.093,2 --11.988,2 --11.884,2 --11.771,2 --11.661,2 --11.548,2 --11.433,2 --11.321,2 --11.214,2 --11.107,2 --10.995,2 --10.89,2 --10.785,2 --10.679,2 --10.555,2 --10.441,2 --10.33,2 --10.224,2 --10.114,2 --10.005,2 --9.892,2 --9.78,2 --9.664,2 --9.555,2 --9.442,2 --9.331,2 --9.214,2 --9.105,2 --8.992,2 --8.889,2 --8.777,2 --8.668,2 --8.558,2 --8.446,2 --8.339,2 --8.229,2 --8.122,2 --8.014,2 --7.908,2 --7.798,2 --7.684,2 --7.57,2 --7.464,2 --7.351,2 --7.242,2 --7.129,2 --7.016,2 --6.904,2 --6.799,2 --6.695,2 --6.579,2 --6.474,2 --6.365,2 --6.259,2 --6.15,2 --6.043,2 --5.932,2 --5.818,2 --5.706,2 --5.6,2 --5.488,2 --5.381,2 --5.276,2 --5.172,2 --5.063,2 --4.953,2 --4.841,2 --4.734,2 --4.628,2 --4.519,2 --4.407,2 --4.293,2 --4.182,2 --4.073,2 --3.963,2 --3.852,2 --3.738,2 --3.634,2 --3.525,2 --3.414,2 --3.301,2 --3.189,2 --3.082,2 --2.971,2 --2.863,2 --2.759,2 --2.649,2 --2.539,2 --2.43,2 --2.315,2 --2.208,2 --2.097,2 --1.987,2 --1.873,2 --1.764,2 --1.656,2 --1.547,2 --1.445,2 --1.329,2 --1.217,2 --1.109,2 --0.995,2 --0.884,2 --0.773,2 --0.661,2 --0.554,2 --0.443,2 --0.329,2 --0.219,2 --0.109,2 --0.0,2 -0.115,2 -0.224,2 -0.331,2 -0.439,2 -0.545,2 -0.651,2 -0.755,2 -0.866,2 -0.975,2 -1.082,2 -1.189,2 -1.296,2 -1.405,1 -1.516,1 -1.621,1 -1.726,1 -1.836,1 -1.948,1 -2.059,1 -2.17,1 -2.284,1 -2.396,1 -2.508,1 -2.62,1 -2.734,1 -2.842,1 -2.954,1 -3.057,1 -3.17,1 -3.281,1 -3.39,1 -3.5,1 -3.61,1 -3.717,1 -3.83,1 -3.94,1 -4.05,1 -4.16,1 -4.271,1 -4.375,1 -4.48,1 -4.594,1 -4.7,1 -4.812,1 -4.918,1 -5.03,1 -5.141,1 -5.251,1 -5.364,1 -5.474,1 -5.587,1 -5.695,1 -5.805,1 -5.909,1 -6.014,1 -6.12,1 -6.23,1 -6.344,1 -6.455,1 -6.568,1 -6.674,1 -6.787,1 -6.893,1 -6.996,1 -7.103,1 -7.212,1 -7.322,1 -7.431,1 -7.543,1 -7.651,1 -7.762,1 -7.872,1 -7.983,1 -8.096,1 -8.206,1 -8.315,1 -8.43,1 -8.536,1 -8.644,1 -8.75,1 -8.861,1 -8.973,1 -9.086,1 -9.196,1 -9.307,1 -9.413,1 -9.527,1 -9.631,1 -9.738,1 -9.85,1 -9.954,1 -10.059,1 -10.172,1 -10.283,1 -10.395,1 -10.504,1 -10.616,1 -10.724,1 -10.832,1 -10.945,1 -11.05,1 -11.159,1 -11.265,1 -11.368,1 -11.472,1 -11.576,1 -11.682,1 -11.793,1 -11.898,1 -12.01,1 -12.123,1 -12.23,1 -12.343,1 -12.452,1 -12.559,1 -12.665,1 -12.774,1 -12.887,1 -12.995,1 -13.103,1 -13.211,1 -13.326,1 -13.431,1 -13.535,1 -13.648,1 -13.753,1 -13.865,1 -13.974,1 -14.085,1 -14.188,1 -14.291,1 -14.395,1 -14.506,1 -14.619,1 -14.727,1 -14.834,2 -14.942,2 -15.054,2 -15.166,2 -15.273,2 -15.398,2 -15.51,2 -15.623,2 -15.737,2 -15.843,2 -15.954,2 -16.065,2 -16.172,2 -16.279,2 -16.382,2 -16.496,2 -16.605,2 -16.708,2 -16.819,2 -16.93,2 -17.038,2 -17.144,2 -17.25,2 -17.361,2 -17.473,2 -17.576,2 -17.684,2 -17.794,2 -17.906,2 -18.021,2 -18.123,2 -18.23,2 -18.337,2 -18.44,2 -18.55,2 -18.654,2 -18.767,2 -18.878,2 -18.987,2 -19.091,2 -19.2,2 -19.307,2 -19.422,2 -19.526,2 -19.636,2 -19.741,2 -19.847,2 -19.959,2 -20.068,2 -20.173,2 -20.285,2 -20.388,2 -20.493,2 -20.602,2 -20.714,2 -20.824,2 -20.93,2 -21.036,2 -21.139,2 -21.251,2 -21.354,2 -21.465,2 -21.575,2 -21.678,2 -21.781,2 -21.886,2 -21.995,2 -22.101,2 -22.206,2 -22.315,2 -22.421,2 -22.526,2 -22.637,2 -22.741,2 -22.856,2 -22.959,2 -23.063,2 -23.174,2 -23.288,2 -23.393,2 -23.5,2 -23.613,2 -23.721,2 -23.831,2 -23.935,2 -24.047,2 -24.155,2 -24.266,2 -24.376,2 -24.483,2 -24.596,2 -24.703,2 -24.816,2 -24.927,2 -25.036,2 -25.147,2 -25.253,2 -25.356,2 -25.464,2 -25.57,2 -25.677,2 -25.789,2 -25.901,2 -26.006,2 -26.113,2 -26.224,2 -26.334,2 -26.446,2 -26.553,2 -26.656,2 -26.759,2 -26.866,2 -26.972,2 -27.078,2 -27.191,2 -27.295,2 -27.405,2 -27.517,2 -27.626,2 -27.734,2 -27.846,2 -27.958,2 -28.069,2 -28.18,2 -28.288,2 -28.395,2 -28.504,2 -28.614,2 -28.724,2 -28.835,2 -28.94,2 -29.051,2 -29.162,2 -29.274,2 -29.392,2 -29.502,2 -29.605,2 -29.713,2 -29.818,2 -29.931,2 -30.042,2 -30.154,2 -30.26,2 -30.367,2 -30.475,2 -30.582,2 -30.69,2 -30.797,2 -30.91,2 -31.016,2 -31.131,2 -31.238,2 -31.351,2 -31.455,2 -31.569,2 -31.678,2 -31.79,2 -31.899,2 -32.01,2 -32.116,2 -32.22,2 -32.324,2 -32.436,2 -32.539,2 -32.642,2 -32.756,2 -32.866,2 -32.978,2 -33.09,2 -33.202,2 -33.312,2 -33.421,2 -33.53,2 -33.639,2 -33.743,2 -33.848,2 -33.959,2 -34.073,2 -34.184,2 -34.29,2 -34.398,2 -34.503,2 -34.614,2 -34.718,2 -34.825,2 -34.938,2 -35.045,2 -35.159,2 -35.265,2 -35.378,2 -35.485,2 -35.591,2 -35.693,2 -35.797,2 -35.908,2 -36.01,2 -36.115,2 -36.218,2 -36.325,2 -36.436,2 -36.549,2 -36.657,2 -36.763,2 -36.872,2 -36.981,2 -37.089,2 -37.198,2 -37.311,2 -37.422,2 -37.526,2 -37.637,2 -37.745,2 -37.848,2 -37.958,2 -38.062,2 -38.164,2 -38.271,2 -38.378,2 -38.486,2 -38.601,2 -38.711,2 -38.822,2 -38.928,2 -39.032,2 -39.142,2 -39.255,2 -39.368,2 -39.475,2 -39.581,2 -39.691,2 -39.795,2 -39.906,2 -40.017,2 -40.123,2 -40.235,2 -40.342,2 -40.452,2 -40.556,2 -40.666,2 -40.776,2 -40.889,2 -41.001,2 -41.112,2 -41.223,2 -41.335,2 -41.441,2 -41.551,2 -41.664,2 -41.776,2 -41.885,2 -41.999,2 -42.105,2 -42.219,2 -42.329,2 -42.44,2 -42.549,2 -42.657,2 -42.762,2 -42.874,2 -42.978,2 -43.087,2 -43.19,2 -43.303,2 -43.406,2 -43.517,2 -43.625,2 -43.734,2 -43.841,2 -43.954,2 -44.067,2 -44.171,2 -44.275,2 -44.38,2 -44.487,2 -44.593,2 -44.704,2 -44.807,2 -44.919,2 -45.032,2 -45.136,2 -45.243,2 -45.352,2 -45.458,2 diff --git a/evidence/failover-each/ff81f93e8b0d/probe.csv b/evidence/failover-each/ff81f93e8b0d/probe.csv deleted file mode 100644 index b15ea058..00000000 --- a/evidence/failover-each/ff81f93e8b0d/probe.csv +++ /dev/null @@ -1,647 +0,0 @@ -t_rel_s,status,latency_ms,instance --25.788,201,19.6,ff81f93e8b0d --25.668,201,5.5,fe75c6c37e5a --25.561,201,15.1,ff81f93e8b0d --25.441,201,5.8,fe75c6c37e5a --25.326,201,4.4,fe75c6c37e5a --25.217,201,3.8,fe75c6c37e5a --25.104,201,3.9,ff81f93e8b0d --24.997,201,3.6,fe75c6c37e5a --24.884,201,5.8,fe75c6c37e5a --24.768,201,5.7,ff81f93e8b0d --24.655,201,3.6,ff81f93e8b0d --24.542,201,5.1,ff81f93e8b0d --24.436,201,4.9,ff81f93e8b0d --24.33,201,3.5,ff81f93e8b0d --24.218,201,5.3,ff81f93e8b0d --24.105,201,5.0,ff81f93e8b0d --23.99,201,6.1,fe75c6c37e5a --23.884,201,4.6,ff81f93e8b0d --23.775,201,4.7,ff81f93e8b0d --23.666,201,4.2,ff81f93e8b0d --23.554,201,3.3,ff81f93e8b0d --23.441,201,4.6,ff81f93e8b0d --23.333,201,3.5,ff81f93e8b0d --23.22,201,4.9,fe75c6c37e5a --23.11,201,4.8,fe75c6c37e5a --23.004,201,4.1,ff81f93e8b0d --22.899,201,4.4,ff81f93e8b0d --22.788,201,5.7,ff81f93e8b0d --22.673,201,3.8,ff81f93e8b0d --22.559,201,4.9,fe75c6c37e5a --22.447,201,3.0,fe75c6c37e5a --22.339,201,4.0,fe75c6c37e5a --22.234,201,4.5,ff81f93e8b0d --22.122,201,6.1,fe75c6c37e5a --22.013,201,4.3,ff81f93e8b0d --21.907,201,3.5,ff81f93e8b0d --21.795,201,5.8,ff81f93e8b0d --21.68,201,5.4,fe75c6c37e5a --21.574,201,3.8,ff81f93e8b0d --21.462,201,5.4,ff81f93e8b0d --21.354,201,4.7,ff81f93e8b0d --21.241,201,5.2,ff81f93e8b0d --21.128,201,4.4,fe75c6c37e5a --21.015,201,5.1,fe75c6c37e5a --20.908,201,4.2,fe75c6c37e5a --20.794,201,3.3,fe75c6c37e5a --20.685,201,4.4,fe75c6c37e5a --20.571,201,3.1,fe75c6c37e5a --20.459,201,4.1,ff81f93e8b0d --20.351,201,4.6,fe75c6c37e5a --20.239,201,5.1,ff81f93e8b0d --20.128,201,4.2,ff81f93e8b0d --20.018,201,3.7,ff81f93e8b0d --19.904,201,5.5,ff81f93e8b0d --19.794,201,3.4,ff81f93e8b0d --19.683,201,4.9,ff81f93e8b0d --19.576,201,4.4,ff81f93e8b0d --19.467,201,4.5,ff81f93e8b0d --19.362,201,4.6,fe75c6c37e5a --19.25,201,5.3,fe75c6c37e5a --19.139,201,3.6,ff81f93e8b0d --19.027,201,3.3,ff81f93e8b0d --18.914,201,4.6,ff81f93e8b0d --18.802,201,5.7,ff81f93e8b0d --18.692,201,4.5,ff81f93e8b0d --18.578,201,4.5,fe75c6c37e5a --18.471,201,5.3,fe75c6c37e5a --18.36,201,3.3,fe75c6c37e5a --18.252,201,3.7,ff81f93e8b0d --18.148,201,3.9,ff81f93e8b0d --18.036,201,4.5,ff81f93e8b0d --17.924,201,5.9,ff81f93e8b0d --17.815,201,6.3,ff81f93e8b0d --17.7,201,5.9,fe75c6c37e5a --17.591,201,4.3,fe75c6c37e5a --17.477,201,3.4,fe75c6c37e5a --17.373,201,3.0,ff81f93e8b0d --17.264,201,5.0,ff81f93e8b0d --17.149,201,5.1,fe75c6c37e5a --17.034,201,4.9,fe75c6c37e5a --16.924,201,4.6,fe75c6c37e5a --16.816,201,4.2,fe75c6c37e5a --16.706,201,3.8,fe75c6c37e5a --16.597,201,3.5,fe75c6c37e5a --16.484,201,3.3,fe75c6c37e5a --16.376,201,4.1,fe75c6c37e5a --16.267,201,6.1,fe75c6c37e5a --16.159,201,4.4,fe75c6c37e5a --16.048,201,5.2,fe75c6c37e5a --15.933,201,3.7,fe75c6c37e5a --15.821,201,3.8,fe75c6c37e5a --15.707,201,2.7,ff81f93e8b0d --15.599,201,2.6,ff81f93e8b0d --15.488,201,4.0,ff81f93e8b0d --15.384,201,3.9,fe75c6c37e5a --15.275,201,3.9,fe75c6c37e5a --15.161,201,3.2,fe75c6c37e5a --15.051,201,3.4,fe75c6c37e5a --14.938,201,4.0,fe75c6c37e5a --14.828,201,2.7,fe75c6c37e5a --14.725,201,2.6,ff81f93e8b0d --14.616,201,3.7,ff81f93e8b0d --14.509,201,4.0,fe75c6c37e5a --14.403,201,5.0,fe75c6c37e5a --14.294,201,5.5,fe75c6c37e5a --14.178,201,4.8,ff81f93e8b0d --14.063,201,4.1,ff81f93e8b0d --13.955,201,4.3,ff81f93e8b0d --13.851,201,4.0,fe75c6c37e5a --13.738,201,5.9,ff81f93e8b0d --13.629,201,4.7,ff81f93e8b0d --13.514,201,6.7,ff81f93e8b0d --13.407,201,5.9,ff81f93e8b0d --13.295,201,5.7,ff81f93e8b0d --13.185,201,4.0,ff81f93e8b0d --13.071,201,2.9,ff81f93e8b0d --12.967,201,3.5,ff81f93e8b0d --12.855,201,4.8,ff81f93e8b0d --12.746,201,4.7,ff81f93e8b0d --12.637,201,4.3,ff81f93e8b0d --12.524,201,5.0,ff81f93e8b0d --12.411,201,3.5,ff81f93e8b0d --12.304,201,4.5,ff81f93e8b0d --12.199,201,5.4,ff81f93e8b0d --12.093,201,4.6,fe75c6c37e5a --11.978,201,3.1,ff81f93e8b0d --11.866,201,4.7,ff81f93e8b0d --11.752,201,5.2,fe75c6c37e5a --11.636,201,5.0,fe75c6c37e5a --11.525,201,4.5,fe75c6c37e5a --11.411,201,7.4,ff81f93e8b0d --11.297,201,3.8,ff81f93e8b0d --11.192,201,2.9,fe75c6c37e5a --11.081,201,4.8,fe75c6c37e5a --10.968,201,4.3,ff81f93e8b0d --10.862,201,3.8,fe75c6c37e5a --10.75,201,5.0,fe75c6c37e5a --10.644,201,3.9,fe75c6c37e5a --10.534,201,8.5,ff81f93e8b0d --10.419,201,4.9,ff81f93e8b0d --10.304,201,6.5,ff81f93e8b0d --10.196,201,3.8,ff81f93e8b0d --10.086,201,4.0,ff81f93e8b0d --9.98,201,14.6,ff81f93e8b0d --9.857,201,10.9,fe75c6c37e5a --9.741,201,10.3,fe75c6c37e5a --9.627,201,6.2,ff81f93e8b0d --9.514,201,8.7,ff81f93e8b0d --9.4,201,5.0,ff81f93e8b0d --9.286,201,4.9,fe75c6c37e5a --9.172,201,4.6,fe75c6c37e5a --9.06,201,6.8,fe75c6c37e5a --8.952,201,6.2,fe75c6c37e5a --8.843,201,7.5,fe75c6c37e5a --8.729,201,6.0,fe75c6c37e5a --8.615,201,9.7,ff81f93e8b0d --8.496,201,8.2,fe75c6c37e5a --8.382,201,11.3,fe75c6c37e5a --8.263,201,5.0,ff81f93e8b0d --8.157,201,5.3,ff81f93e8b0d --8.048,201,6.2,ff81f93e8b0d --7.936,201,5.7,ff81f93e8b0d --7.824,201,8.2,ff81f93e8b0d --7.707,201,6.9,fe75c6c37e5a --7.593,201,6.7,fe75c6c37e5a --7.482,201,4.9,fe75c6c37e5a --7.373,201,5.1,fe75c6c37e5a --7.26,201,12.5,ff81f93e8b0d --7.144,201,8.6,fe75c6c37e5a --7.027,201,8.1,fe75c6c37e5a --6.918,201,5.2,fe75c6c37e5a --6.806,201,6.4,fe75c6c37e5a --6.699,201,5.1,fe75c6c37e5a --6.583,201,4.1,fe75c6c37e5a --6.474,201,5.0,fe75c6c37e5a --6.364,201,3.3,fe75c6c37e5a --6.259,201,4.7,fe75c6c37e5a --6.152,201,9.2,fe75c6c37e5a --6.034,201,4.7,fe75c6c37e5a --5.92,201,8.7,ff81f93e8b0d --5.806,201,4.8,ff81f93e8b0d --5.699,201,5.2,fe75c6c37e5a --5.591,201,4.2,fe75c6c37e5a --5.481,201,3.3,fe75c6c37e5a --5.371,201,3.5,fe75c6c37e5a --5.258,201,7.0,ff81f93e8b0d --5.142,201,7.8,ff81f93e8b0d --5.026,201,5.2,fe75c6c37e5a --4.92,201,5.1,fe75c6c37e5a --4.804,201,4.2,fe75c6c37e5a --4.692,201,5.1,fe75c6c37e5a --4.586,201,3.6,fe75c6c37e5a --4.478,201,5.5,fe75c6c37e5a --4.366,201,6.3,fe75c6c37e5a --4.256,201,4.2,fe75c6c37e5a --4.146,201,9.5,fe75c6c37e5a --4.028,201,6.7,ff81f93e8b0d --3.913,201,5.2,ff81f93e8b0d --3.8,201,6.4,fe75c6c37e5a --3.69,201,5.3,fe75c6c37e5a --3.574,201,4.6,fe75c6c37e5a --3.464,201,3.3,fe75c6c37e5a --3.355,201,3.7,fe75c6c37e5a --3.243,201,7.0,fe75c6c37e5a --3.13,201,5.2,fe75c6c37e5a --3.02,201,5.5,fe75c6c37e5a --2.908,201,5.6,ff81f93e8b0d --2.803,201,5.7,fe75c6c37e5a --2.687,201,3.8,ff81f93e8b0d --2.58,201,4.9,ff81f93e8b0d --2.471,201,5.7,ff81f93e8b0d --2.356,201,4.9,ff81f93e8b0d --2.251,201,6.1,ff81f93e8b0d --2.137,201,5.0,ff81f93e8b0d --2.025,201,5.3,ff81f93e8b0d --1.918,201,4.7,ff81f93e8b0d --1.805,201,3.6,ff81f93e8b0d --1.7,201,5.7,fe75c6c37e5a --1.595,201,4.7,fe75c6c37e5a --1.484,201,5.2,fe75c6c37e5a --1.369,201,5.7,ff81f93e8b0d --1.255,201,6.0,ff81f93e8b0d --1.149,201,6.1,ff81f93e8b0d --1.034,201,5.9,ff81f93e8b0d --0.921,201,4.7,ff81f93e8b0d --0.808,201,3.3,fe75c6c37e5a --0.702,201,4.7,ff81f93e8b0d --0.588,201,4.7,fe75c6c37e5a --0.474,201,5.5,fe75c6c37e5a --0.361,201,4.5,fe75c6c37e5a --0.253,201,6.2,fe75c6c37e5a --0.141,201,5.0,fe75c6c37e5a --0.027,201,5.0,ff81f93e8b0d -0.087,201,6.4,ff81f93e8b0d -0.203,201,3.1,fe75c6c37e5a -0.315,201,4.3,fe75c6c37e5a -0.427,201,4.6,fe75c6c37e5a -0.535,201,7.2,fe75c6c37e5a -0.642,201,4.4,fe75c6c37e5a -0.757,201,4.6,fe75c6c37e5a -0.864,201,4.0,fe75c6c37e5a -0.976,201,2.8,fe75c6c37e5a -1.082,201,3.6,fe75c6c37e5a -1.19,201,5.3,ff81f93e8b0d -1.302,201,5.9,fe75c6c37e5a -1.408,201,4.5,fe75c6c37e5a -1.521,201,4.6,fe75c6c37e5a -1.626,201,2.8,fe75c6c37e5a -1.728,201,3.7,fe75c6c37e5a -1.84,201,3.8,fe75c6c37e5a -1.947,201,3.5,fe75c6c37e5a -2.05,201,3.6,fe75c6c37e5a -2.156,201,4.9,fe75c6c37e5a -2.265,201,4.4,fe75c6c37e5a -2.377,201,13.1,fe75c6c37e5a -2.491,201,28.9,fe75c6c37e5a -2.622,201,9.4,fe75c6c37e5a -2.739,201,5.0,fe75c6c37e5a -2.845,201,9.4,fe75c6c37e5a -2.962,201,3.2,fe75c6c37e5a -3.066,201,4.3,fe75c6c37e5a -3.176,201,3.8,fe75c6c37e5a -3.283,201,7.4,fe75c6c37e5a -3.393,201,4.2,fe75c6c37e5a -3.507,201,4.6,fe75c6c37e5a -3.619,201,4.6,fe75c6c37e5a -3.733,201,4.3,fe75c6c37e5a -3.842,201,5.3,fe75c6c37e5a -3.957,201,6.3,fe75c6c37e5a -4.067,201,4.0,fe75c6c37e5a -4.178,201,4.3,fe75c6c37e5a -4.285,201,4.7,fe75c6c37e5a -4.396,201,4.0,fe75c6c37e5a -4.503,201,8.8,fe75c6c37e5a -4.616,201,11.8,fe75c6c37e5a -4.735,201,3.9,fe75c6c37e5a -4.847,201,14.6,fe75c6c37e5a -4.971,201,5.8,fe75c6c37e5a -5.08,201,6.2,fe75c6c37e5a -5.194,201,3.7,fe75c6c37e5a -5.306,201,6.6,fe75c6c37e5a -5.417,201,5.8,fe75c6c37e5a -5.533,201,4.8,fe75c6c37e5a -5.639,201,5.3,fe75c6c37e5a -5.753,201,3.2,fe75c6c37e5a -5.857,201,6.7,fe75c6c37e5a -5.973,201,6.9,fe75c6c37e5a -6.088,201,5.5,fe75c6c37e5a -6.202,201,5.7,fe75c6c37e5a -6.315,201,5.0,fe75c6c37e5a -6.425,201,5.2,fe75c6c37e5a -6.537,201,4.8,fe75c6c37e5a -6.646,201,3.3,fe75c6c37e5a -6.755,201,6.7,fe75c6c37e5a -6.868,201,4.5,fe75c6c37e5a -6.979,201,4.5,fe75c6c37e5a -7.088,201,5.4,fe75c6c37e5a -7.199,201,4.9,fe75c6c37e5a -7.312,201,5.1,fe75c6c37e5a -7.42,201,4.4,fe75c6c37e5a -7.534,201,4.6,fe75c6c37e5a -7.647,201,3.4,fe75c6c37e5a -7.755,201,4.9,fe75c6c37e5a -7.867,201,3.3,fe75c6c37e5a -7.974,201,6.0,fe75c6c37e5a -8.088,201,6.2,fe75c6c37e5a -8.203,201,11.6,fe75c6c37e5a -8.317,201,4.0,fe75c6c37e5a -8.426,201,4.0,fe75c6c37e5a -8.538,201,2.6,fe75c6c37e5a -8.645,201,4.8,fe75c6c37e5a -8.756,201,8.3,fe75c6c37e5a -8.869,201,5.0,fe75c6c37e5a -8.978,201,4.5,fe75c6c37e5a -9.087,201,3.8,fe75c6c37e5a -9.201,201,4.7,fe75c6c37e5a -9.311,201,4.1,fe75c6c37e5a -9.423,201,5.8,fe75c6c37e5a -9.538,201,2.8,fe75c6c37e5a -9.647,201,3.0,fe75c6c37e5a -9.76,201,4.3,fe75c6c37e5a -9.872,201,6.4,fe75c6c37e5a -9.984,201,3.7,fe75c6c37e5a -10.089,201,7.5,fe75c6c37e5a -10.199,201,5.3,fe75c6c37e5a -10.305,201,6.6,fe75c6c37e5a -10.421,201,3.1,fe75c6c37e5a -10.534,201,3.0,fe75c6c37e5a -10.647,201,3.1,fe75c6c37e5a -10.752,201,4.8,fe75c6c37e5a -10.867,201,4.4,fe75c6c37e5a -10.974,201,6.7,fe75c6c37e5a -11.084,201,5.7,fe75c6c37e5a -11.191,201,6.4,fe75c6c37e5a -11.303,201,3.2,fe75c6c37e5a -11.407,201,4.9,fe75c6c37e5a -11.515,201,5.0,fe75c6c37e5a -11.628,201,4.7,fe75c6c37e5a -11.742,201,4.7,fe75c6c37e5a -11.851,201,6.6,fe75c6c37e5a -11.966,201,5.4,fe75c6c37e5a -12.076,201,4.1,fe75c6c37e5a -12.19,201,7.0,fe75c6c37e5a -12.299,201,4.9,fe75c6c37e5a -12.414,201,4.8,fe75c6c37e5a -12.52,201,7.8,fe75c6c37e5a -12.637,201,3.5,fe75c6c37e5a -12.746,201,3.7,fe75c6c37e5a -12.854,201,3.4,fe75c6c37e5a -12.961,201,3.7,fe75c6c37e5a -13.069,201,3.8,fe75c6c37e5a -13.183,201,3.7,fe75c6c37e5a -13.293,201,5.8,fe75c6c37e5a -13.402,201,7.9,fe75c6c37e5a -13.513,201,6.1,fe75c6c37e5a -13.626,201,3.7,fe75c6c37e5a -13.736,201,4.1,fe75c6c37e5a -13.848,201,3.5,fe75c6c37e5a -13.96,201,3.2,fe75c6c37e5a -14.07,201,3.6,fe75c6c37e5a -14.183,201,3.4,fe75c6c37e5a -14.29,201,4.4,fe75c6c37e5a -14.397,201,3.1,fe75c6c37e5a -14.506,201,3.5,fe75c6c37e5a -14.615,201,3.5,fe75c6c37e5a -14.72,201,3.4,fe75c6c37e5a -14.826,201,3.6,fe75c6c37e5a -14.93,201,4.2,ff81f93e8b0d -15.034,201,3.8,ff81f93e8b0d -15.147,201,3.6,ff81f93e8b0d -15.261,201,3.7,fe75c6c37e5a -15.366,201,3.4,ff81f93e8b0d -15.474,201,3.4,ff81f93e8b0d -15.586,201,5.0,ff81f93e8b0d -15.701,201,3.7,ff81f93e8b0d -15.813,201,3.9,fe75c6c37e5a -15.926,201,4.0,fe75c6c37e5a -16.04,201,6.1,fe75c6c37e5a -16.157,201,5.0,ff81f93e8b0d -16.265,201,3.9,ff81f93e8b0d -16.375,201,14.9,ff81f93e8b0d -16.494,201,4.5,fe75c6c37e5a -16.599,201,4.6,ff81f93e8b0d -16.708,201,5.7,ff81f93e8b0d -16.815,201,4.9,ff81f93e8b0d -16.922,201,6.2,ff81f93e8b0d -17.029,201,4.6,fe75c6c37e5a -17.135,201,4.3,fe75c6c37e5a -17.241,201,4.7,fe75c6c37e5a -17.353,201,4.6,fe75c6c37e5a -17.468,201,4.0,fe75c6c37e5a -17.575,201,4.3,fe75c6c37e5a -17.685,201,2.8,fe75c6c37e5a -17.792,201,5.1,fe75c6c37e5a -17.904,201,3.7,fe75c6c37e5a -18.018,201,4.0,fe75c6c37e5a -18.126,201,3.3,fe75c6c37e5a -18.23,201,4.0,ff81f93e8b0d -18.34,201,5.9,ff81f93e8b0d -18.455,201,5.9,fe75c6c37e5a -18.571,201,3.5,fe75c6c37e5a -18.678,201,5.3,fe75c6c37e5a -18.791,201,5.5,ff81f93e8b0d -18.899,201,3.0,fe75c6c37e5a -19.01,201,3.8,fe75c6c37e5a -19.116,201,5.1,fe75c6c37e5a -19.224,201,5.0,fe75c6c37e5a -19.336,201,3.7,fe75c6c37e5a -19.446,201,3.4,fe75c6c37e5a -19.555,201,10.6,fe75c6c37e5a -19.672,201,6.1,ff81f93e8b0d -19.787,201,7.7,ff81f93e8b0d -19.905,201,7.1,fe75c6c37e5a -20.02,201,7.8,ff81f93e8b0d -20.132,201,4.3,ff81f93e8b0d -20.237,201,6.4,fe75c6c37e5a -20.35,201,3.7,fe75c6c37e5a -20.462,201,4.3,ff81f93e8b0d -20.574,201,3.3,ff81f93e8b0d -20.679,201,3.1,fe75c6c37e5a -20.784,201,3.5,fe75c6c37e5a -20.894,201,4.0,fe75c6c37e5a -21.006,201,4.4,fe75c6c37e5a -21.115,201,3.7,fe75c6c37e5a -21.222,201,5.5,fe75c6c37e5a -21.336,201,3.6,fe75c6c37e5a -21.44,201,6.4,ff81f93e8b0d -21.556,201,5.8,fe75c6c37e5a -21.671,201,4.8,ff81f93e8b0d -21.783,201,2.6,ff81f93e8b0d -21.893,201,4.7,ff81f93e8b0d -21.999,201,3.1,fe75c6c37e5a -22.104,201,2.7,fe75c6c37e5a -22.207,201,5.7,ff81f93e8b0d -22.319,201,2.9,ff81f93e8b0d -22.422,201,5.5,fe75c6c37e5a -22.534,201,4.2,ff81f93e8b0d -22.638,201,5.8,fe75c6c37e5a -22.745,201,2.8,fe75c6c37e5a -22.858,201,4.4,fe75c6c37e5a -22.967,201,3.1,fe75c6c37e5a -23.076,201,3.2,fe75c6c37e5a -23.185,201,3.6,fe75c6c37e5a -23.292,201,3.9,fe75c6c37e5a -23.403,201,3.7,fe75c6c37e5a -23.516,201,3.8,fe75c6c37e5a -23.625,201,4.3,fe75c6c37e5a -23.731,201,2.7,fe75c6c37e5a -23.844,201,3.3,fe75c6c37e5a -23.957,201,5.7,fe75c6c37e5a -24.064,201,3.4,fe75c6c37e5a -24.172,201,5.3,fe75c6c37e5a -24.285,201,2.9,fe75c6c37e5a -24.392,201,5.5,fe75c6c37e5a -24.501,201,3.8,fe75c6c37e5a -24.611,201,3.3,ff81f93e8b0d -24.719,201,5.0,ff81f93e8b0d -24.824,201,4.2,ff81f93e8b0d -24.93,201,2.8,fe75c6c37e5a -25.042,201,4.8,ff81f93e8b0d -25.151,201,3.1,ff81f93e8b0d -25.261,201,3.1,fe75c6c37e5a -25.365,201,3.1,fe75c6c37e5a -25.477,201,4.1,fe75c6c37e5a -25.588,201,3.3,fe75c6c37e5a -25.691,201,4.4,fe75c6c37e5a -25.797,201,3.1,ff81f93e8b0d -25.904,201,2.8,ff81f93e8b0d -26.011,201,3.1,fe75c6c37e5a -26.124,201,5.2,ff81f93e8b0d -26.239,201,5.3,ff81f93e8b0d -26.35,201,3.1,fe75c6c37e5a -26.46,201,4.1,fe75c6c37e5a -26.57,201,3.5,ff81f93e8b0d -26.683,201,3.1,fe75c6c37e5a -26.788,201,4.1,ff81f93e8b0d -26.893,201,3.0,ff81f93e8b0d -26.996,201,4.1,fe75c6c37e5a -27.109,201,4.0,fe75c6c37e5a -27.22,201,3.2,fe75c6c37e5a -27.325,201,4.6,fe75c6c37e5a -27.431,201,2.5,fe75c6c37e5a -27.536,201,2.9,ff81f93e8b0d -27.641,201,4.3,ff81f93e8b0d -27.747,201,3.2,fe75c6c37e5a -27.86,201,3.3,ff81f93e8b0d -27.972,201,4.7,fe75c6c37e5a -28.084,201,2.8,ff81f93e8b0d -28.197,201,4.0,ff81f93e8b0d -28.302,201,3.3,ff81f93e8b0d -28.408,201,2.9,fe75c6c37e5a -28.517,201,2.8,fe75c6c37e5a -28.625,201,4.3,fe75c6c37e5a -28.737,201,3.4,fe75c6c37e5a -28.842,201,3.2,fe75c6c37e5a -28.953,201,3.1,fe75c6c37e5a -29.065,201,2.9,fe75c6c37e5a -29.178,201,5.1,fe75c6c37e5a -29.291,201,3.5,ff81f93e8b0d -29.404,201,11.5,ff81f93e8b0d -29.517,201,3.0,ff81f93e8b0d -29.625,201,2.8,ff81f93e8b0d -29.737,201,3.3,ff81f93e8b0d -29.841,201,4.1,fe75c6c37e5a -29.954,201,4.0,ff81f93e8b0d -30.062,201,4.1,ff81f93e8b0d -30.167,201,2.9,fe75c6c37e5a -30.273,201,3.6,fe75c6c37e5a -30.387,201,3.5,fe75c6c37e5a -30.498,201,3.1,fe75c6c37e5a -30.602,201,3.8,fe75c6c37e5a -30.707,201,4.5,ff81f93e8b0d -30.822,201,3.8,fe75c6c37e5a -30.932,201,3.9,fe75c6c37e5a -31.045,201,7.3,fe75c6c37e5a -31.158,201,5.1,fe75c6c37e5a -31.271,201,3.7,ff81f93e8b0d -31.381,201,4.5,ff81f93e8b0d -31.491,201,3.9,ff81f93e8b0d -31.598,201,3.0,fe75c6c37e5a -31.706,201,3.5,fe75c6c37e5a -31.815,201,4.9,fe75c6c37e5a -31.925,201,2.9,fe75c6c37e5a -32.032,201,3.4,fe75c6c37e5a -32.144,201,4.1,fe75c6c37e5a -32.249,201,5.3,ff81f93e8b0d -32.364,201,3.2,fe75c6c37e5a -32.475,201,3.3,fe75c6c37e5a -32.579,201,3.6,ff81f93e8b0d -32.691,201,3.1,ff81f93e8b0d -32.797,201,4.3,ff81f93e8b0d -32.91,201,3.5,ff81f93e8b0d -33.018,201,3.2,ff81f93e8b0d -33.131,201,2.9,fe75c6c37e5a -33.236,201,4.3,ff81f93e8b0d -33.343,201,3.5,ff81f93e8b0d -33.453,201,3.2,ff81f93e8b0d -33.558,201,3.4,fe75c6c37e5a -33.671,201,3.4,ff81f93e8b0d -33.776,201,4.7,fe75c6c37e5a -33.889,201,3.8,fe75c6c37e5a -34.002,201,4.0,ff81f93e8b0d -34.114,201,3.8,ff81f93e8b0d -34.228,201,3.0,ff81f93e8b0d -34.333,201,3.5,ff81f93e8b0d -34.437,201,3.7,fe75c6c37e5a -34.543,201,4.2,fe75c6c37e5a -34.657,201,4.4,fe75c6c37e5a -34.77,201,4.5,ff81f93e8b0d -34.882,201,5.8,ff81f93e8b0d -34.995,201,4.4,ff81f93e8b0d -35.107,201,3.4,ff81f93e8b0d -35.215,201,5.2,ff81f93e8b0d -35.328,201,4.3,ff81f93e8b0d -35.443,201,3.8,fe75c6c37e5a -35.557,201,16.2,fe75c6c37e5a -35.68,201,3.3,fe75c6c37e5a -35.786,201,3.3,ff81f93e8b0d -35.897,201,3.2,ff81f93e8b0d -36.01,201,3.6,ff81f93e8b0d -36.116,201,2.7,ff81f93e8b0d -36.227,201,4.7,ff81f93e8b0d -36.335,201,2.7,ff81f93e8b0d -36.442,201,3.0,ff81f93e8b0d -36.55,201,3.2,fe75c6c37e5a -36.654,201,3.4,fe75c6c37e5a -36.763,201,3.5,fe75c6c37e5a -36.872,201,3.4,fe75c6c37e5a -36.981,201,5.0,ff81f93e8b0d -37.089,201,5.0,ff81f93e8b0d -37.199,201,2.9,ff81f93e8b0d -37.305,201,3.6,ff81f93e8b0d -37.415,201,3.7,fe75c6c37e5a -37.521,201,5.5,ff81f93e8b0d -37.632,201,4.0,ff81f93e8b0d -37.744,201,3.7,ff81f93e8b0d -37.858,201,3.5,ff81f93e8b0d -37.966,201,2.8,ff81f93e8b0d -38.074,201,2.9,ff81f93e8b0d -38.185,201,3.6,ff81f93e8b0d -38.297,201,5.1,ff81f93e8b0d -38.407,201,3.5,ff81f93e8b0d -38.52,201,3.1,ff81f93e8b0d -38.632,201,3.4,ff81f93e8b0d -38.744,201,3.0,fe75c6c37e5a -38.849,201,3.6,ff81f93e8b0d -38.954,201,3.2,ff81f93e8b0d -39.062,201,3.2,ff81f93e8b0d -39.168,201,3.3,fe75c6c37e5a -39.276,201,2.7,fe75c6c37e5a -39.388,201,2.9,fe75c6c37e5a -39.497,201,3.0,fe75c6c37e5a -39.606,201,3.6,fe75c6c37e5a -39.714,201,2.9,fe75c6c37e5a -39.82,201,3.2,ff81f93e8b0d -39.932,201,2.9,fe75c6c37e5a -40.043,201,3.4,fe75c6c37e5a -40.155,201,7.2,fe75c6c37e5a -40.262,201,3.0,fe75c6c37e5a -40.368,201,2.7,ff81f93e8b0d -40.476,201,3.7,ff81f93e8b0d -40.584,201,3.7,ff81f93e8b0d -40.691,201,4.9,fe75c6c37e5a -40.805,201,4.2,ff81f93e8b0d -40.911,201,3.8,ff81f93e8b0d -41.015,201,2.7,fe75c6c37e5a -41.125,201,3.0,fe75c6c37e5a -41.234,201,3.1,fe75c6c37e5a -41.339,201,2.7,ff81f93e8b0d -41.45,201,2.7,ff81f93e8b0d -41.56,201,2.6,ff81f93e8b0d -41.665,201,3.8,ff81f93e8b0d -41.775,201,4.8,ff81f93e8b0d -41.885,201,6.7,ff81f93e8b0d -41.997,201,5.8,ff81f93e8b0d -42.105,201,5.2,ff81f93e8b0d -42.216,201,4.7,ff81f93e8b0d -42.326,201,4.1,ff81f93e8b0d -42.44,201,5.0,ff81f93e8b0d -42.553,201,3.3,fe75c6c37e5a -42.657,201,3.5,ff81f93e8b0d -42.762,201,4.0,ff81f93e8b0d -42.875,201,2.7,ff81f93e8b0d -42.986,201,2.9,ff81f93e8b0d -43.096,201,2.8,ff81f93e8b0d -43.202,201,3.7,ff81f93e8b0d -43.316,201,2.8,ff81f93e8b0d -43.426,201,3.4,ff81f93e8b0d -43.537,201,3.1,ff81f93e8b0d -43.649,201,3.6,ff81f93e8b0d -43.763,201,3.5,fe75c6c37e5a -43.875,201,3.3,fe75c6c37e5a -43.98,201,4.2,ff81f93e8b0d -44.087,201,3.8,ff81f93e8b0d -44.191,201,3.8,ff81f93e8b0d -44.304,201,6.3,ff81f93e8b0d -44.415,201,7.9,ff81f93e8b0d -44.523,201,3.5,ff81f93e8b0d -44.63,201,20.6,ff81f93e8b0d -44.759,201,3.5,fe75c6c37e5a -44.864,201,3.7,fe75c6c37e5a -44.974,201,4.8,fe75c6c37e5a -45.083,201,3.6,ff81f93e8b0d -45.19,201,5.5,fe75c6c37e5a -45.297,201,4.3,ff81f93e8b0d -45.406,201,6.9,fe75c6c37e5a diff --git a/evidence/failover-each/summary.json b/evidence/failover-each/summary.json index 69b68b88..3051dfde 100644 --- a/evidence/failover-each/summary.json +++ b/evidence/failover-each/summary.json @@ -1,122 +1,142 @@ -[ - { - "label": "ff81f93e8b0d", - "action": [ - "sh", - "-c", - "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' api && { scripts/stand-usage.sh & sleep 10; wait; } && scripts/restore.sh" - ], - "action_exit": 0, - "action_seconds": 15.296, - "action_finished_before_load_end": true, - "lb_up_before": 2, - "lb_up_min": 1, - "detection_seconds": 1.405, - "rejoin_seconds": 14.834, - "probe": { - "attempts": 646, - "failed_attempts": 0, - "first_failure_s": null, - "recovery_s": 0.0, - "instances_before": { - "ff81f93e8b0d": 116, - "fe75c6c37e5a": 117 - }, - "instances_during_outage": { - "fe75c6c37e5a": 121 - }, - "instances_after": { - "ff81f93e8b0d": 137, - "fe75c6c37e5a": 138 - } +[{"date": "2026-11-05T19:18:13.470607+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", + "label": "7cfd91ddf780", + "action": [ + "sh", + "-c", + "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' api && { scripts/stand-usage.sh & sleep 10; wait; } && scripts/restore.sh" + ], + "action_exit": 0, + "action_seconds": 15.204, + "action_finished_before_load_end": true, + "lb_up_before": 2, + "lb_up_min": 1, + "lb_samples_missing": 0, + "detection_seconds": 0.884, + "rejoin_seconds": 14.885, + "probe": { + "attempts": 645, + "failed_attempts": 0, + "first_failure_s": null, + "first_success_s": 0.023, + "recovery_s": 0.0, + "instances_before": { + "5bbf259ff136": 118, + "7cfd91ddf780": 115 }, - "lb_counters": { - "wretr": 0, - "wredis": 0, - "econ": 0, - "eresp": 0, - "hrsp_5xx": 0 + "instances_during_outage": { + "5bbf259ff136": 126 }, - "k6": { - "requests": 6001, - "rps": 100.02, - "ok_rps": 100.02, - "failed_ratio": 0, - "dropped_iterations": 0, - "p50_ms": 1.6, - "p95_ms": 3.92, - "p99_ms": 7.31, - "exit": 0, - "rate": 100 - }, - "checks": { - "action_ok": true, - "load_errors_ok": true, - "write_recovery_ok": true, - "detected": true, - "target_excluded": true, - "rejoined": true - }, - "pass": true + "instances_after": { + "5bbf259ff136": 142, + "7cfd91ddf780": 132 + } }, - { - "label": "fe75c6c37e5a", - "action": [ - "sh", - "-c", - "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' api && { scripts/stand-usage.sh & sleep 10; wait; } && scripts/restore.sh" - ], - "action_exit": 0, - "action_seconds": 14.56, - "action_finished_before_load_end": true, - "lb_up_before": 2, - "lb_up_min": 1, - "detection_seconds": 0.859, - "rejoin_seconds": 14.343, - "probe": { - "attempts": 646, - "failed_attempts": 0, - "first_failure_s": null, - "recovery_s": 0.0, - "instances_before": { - "fe75c6c37e5a": 130, - "ff81f93e8b0d": 105 - }, - "instances_during_outage": { - "ff81f93e8b0d": 121 - }, - "instances_after": { - "ff81f93e8b0d": 123, - "fe75c6c37e5a": 156 - } + "history": { + "confirmed": 645, + "stored": 645, + "exact_match": true, + "unique": true + }, + "lb_counters": { + "wretr": 0, + "wredis": 0, + "econ": 0, + "eresp": 0, + "hrsp_5xx": 0 + }, + "k6": { + "requests": 6001, + "rps": 100.02, + "ok_rps": 100.02, + "failed_ratio": 0, + "dropped_iterations": 0, + "p50_ms": 2.97, + "p95_ms": 3.86, + "p99_ms": 4.47, + "exit": 0, + "rate": 100 + }, + "checks": { + "action_ok": true, + "load_errors_ok": true, + "load_delivered": true, + "lb_observed": true, + "write_recovery_ok": true, + "history_preserved": true, + "detected": true, + "target_excluded": true, + "rejoined": true + }, + "pass": true +}, +{"date": "2026-11-05T19:18:13.470607+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", + "label": "5bbf259ff136", + "action": [ + "sh", + "-c", + "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' api && { scripts/stand-usage.sh & sleep 10; wait; } && scripts/restore.sh" + ], + "action_exit": 0, + "action_seconds": 15.2, + "action_finished_before_load_end": true, + "lb_up_before": 2, + "lb_up_min": 1, + "lb_samples_missing": 0, + "detection_seconds": 1.498, + "rejoin_seconds": 14.99, + "probe": { + "attempts": 642, + "failed_attempts": 0, + "first_failure_s": null, + "first_success_s": 0.111, + "recovery_s": 0.0, + "instances_before": { + "7cfd91ddf780": 114, + "5bbf259ff136": 119 }, - "lb_counters": { - "wretr": 0, - "wredis": 0, - "econ": 0, - "eresp": 0, - "hrsp_5xx": 0 + "instances_during_outage": { + "7cfd91ddf780": 121 }, - "k6": { - "requests": 6001, - "rps": 100.02, - "ok_rps": 100.02, - "failed_ratio": 0, - "dropped_iterations": 0, - "p50_ms": 1.57, - "p95_ms": 3.62, - "p99_ms": 6.24, - "exit": 0, - "rate": 100 - }, - "checks": { - "action_ok": true, - "load_errors_ok": true, - "write_recovery_ok": true, - "detected": true, - "target_excluded": true, - "rejoined": true - }, - "pass": true - } -] + "instances_after": { + "7cfd91ddf780": 149, + "5bbf259ff136": 124 + } + }, + "history": { + "confirmed": 642, + "stored": 642, + "exact_match": true, + "unique": true + }, + "lb_counters": { + "wretr": 0, + "wredis": 0, + "econ": 0, + "eresp": 0, + "hrsp_5xx": 0 + }, + "k6": { + "requests": 6001, + "rps": 100.02, + "ok_rps": 100.02, + "failed_ratio": 0, + "dropped_iterations": 0, + "p50_ms": 2.98, + "p95_ms": 3.88, + "p99_ms": 4.53, + "exit": 0, + "rate": 100 + }, + "checks": { + "action_ok": true, + "load_errors_ok": true, + "load_delivered": true, + "lb_observed": true, + "write_recovery_ok": true, + "history_preserved": true, + "detected": true, + "target_excluded": true, + "rejoined": true + }, + "pass": true +}] diff --git a/evidence/grade4-baseline-proxy-failed.txt b/evidence/grade4-baseline-proxy-failed.txt new file mode 100644 index 00000000..6cb7e253 --- /dev/null +++ b/evidence/grade4-baseline-proxy-failed.txt @@ -0,0 +1,8 @@ +date: 2026-11-05T19:16:05.293964+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ sh -ec 'make test; make limits; make lb-check; docker compose ps -a --format "{{.ID}} {{.Service}} {{.Status}}"; docker inspect -f "{{.Name}} {{.Config.Hostname}} image={{.Image}}" $(docker compose ps -q api)' +python3 tests/smoke.py --base-url "http://127.0.0.1:18083" +FAIL: GET /health/live: connection/TLS failure (RemoteDisconnected) +make: *** [test] Error 1 + +exit: 2 elapsed_seconds: 0.276 diff --git a/evidence/grade4-baseline.txt b/evidence/grade4-baseline.txt new file mode 100644 index 00000000..133271b3 --- /dev/null +++ b/evidence/grade4-baseline.txt @@ -0,0 +1,66 @@ +date: 2026-11-05T19:16:58.984609+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ sh -ec 'make restore; make test; make limits; make lb-check; docker compose ps -a --format "{{.ID}} {{.Service}} {{.Status}}"; docker inspect -f "{{.Name}} {{.Config.Hostname}} image={{.Image}}" $(docker compose ps -q api)' +scripts/restore.sh + Container mtusi2026lab3-grade4-20261105-api-1 Running + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +python3 tests/smoke.py --base-url "http://127.0.0.1:18083" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. +python3 scripts/limits.py --out evidence/limits.txt +date: 2026-11-05T19:17:00Z commit: 252e3e3 +$ docker compose --profile bench config --format json (declared) +service cpus/mem ports secrets volumes networks ro +-------- ---------- ------------------------------- ------------------------------------ ---------- ---------------------- ----- +api 1/512MiB - cache_url,cursor_secret,database_url - backend(internal) true +k6 2/512MiB - - - edge false +lb 1/128MiB 127.0.0.1:18083,127.0.0.1:23083 - - backend(internal),edge true +migrate 0.5/256MiB - database_url - backend(internal) false +postgres 2/1024MiB - postgres_password pgdata backend(internal) false +valkey 0.5/128MiB - valkey.conf valkeydata backend(internal) false + +$ docker inspect (running containers: hostname = X-Instance-Id, applied limits) +service hostname cpus memory published +-------- ------------ ---- ------- --------------------------------------------------- +api 5bbf259ff136 1 512MiB +api 7cfd91ddf780 1 512MiB +lb 4f83577b3411 1 128MiB 127.0.0.1:18083->8080/tcp,127.0.0.1:23083->8404/tcp +postgres f225c6cd77bb 2 1024MiB +valkey 5e3ed0a9a35a 0.5 128MiB + +PASS: limits, ports, secrets, volumes and networks match the policy +python3 scripts/probe.py check --rounds 20 --out evidence/lb-check.json +{ + "instances_one_token": { + "7cfd91ddf780": 20, + "5bbf259ff136": 20 + }, + "same_key": { + "rounds": 20, + "parallel": 8, + "statuses": { + "201": 160 + }, + "instances": { + "5bbf259ff136": 80, + "7cfd91ddf780": 80 + }, + "identical_bodies_rounds": 20, + "single_copy_rounds": 20, + "conflict_rounds": 20 + }, + "pass": true +} +7cfd91ddf780 api Up 6 seconds (healthy) +5bbf259ff136 api Up About a minute (healthy) +4f83577b3411 lb Up About a minute (healthy) +ad30cd539e85 migrate Exited (0) About a minute ago +f225c6cd77bb postgres Up About a minute (healthy) +5e3ed0a9a35a valkey Up About a minute (healthy) +/mtusi2026lab3-grade4-20261105-api-1 7cfd91ddf780 image=sha256:fdc0128530c7f092746491b339125eea9fe079b86dcc5c8c3aa89e224fca1706 +/mtusi2026lab3-grade4-20261105-api-2 5bbf259ff136 image=sha256:fdc0128530c7f092746491b339125eea9fe079b86dcc5c8c3aa89e224fca1706 + +exit: 0 elapsed_seconds: 2.654 diff --git a/evidence/grade4-checks.txt b/evidence/grade4-checks.txt new file mode 100644 index 00000000..26838c0e --- /dev/null +++ b/evidence/grade4-checks.txt @@ -0,0 +1,35 @@ +date: 2026-11-05T19:29:36.759200+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ sh -ec 'make check; make validate; make lint; uv lock --check; openspec validate --all --strict; git diff --check; test -z "$(git diff ef0b9a3 --name-only -- contracts tests README.md COURSE.md .pipeline)"; test -z "$(git ls-files .env secrets volumes)"; echo "PASS: normative files and pipeline unchanged; no tracked secrets or volumes"' +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +.venv/bin/ruff check src student-tests alembic scripts/probe.py scripts/limits.py scripts/scaling.py tools +All checks passed! +.venv/bin/ruff format --check src student-tests alembic scripts/probe.py scripts/limits.py scripts/scaling.py tools +108 files already formatted +.venv/bin/ty check src student-tests alembic +All checks passed! +Resolved 64 packages in 18ms +- Validating... +✓ change/2026-11-05-adopt-sqlalchemy-alembic +✓ spec/lab1-event-publication +✓ spec/lab1-messenger-chats +✓ spec/lab1-messenger-history +✓ spec/lab1-messenger-identity +✓ spec/lab1-service-operability +✓ spec/lab2-auth-sessions +✓ spec/lab2-dependency-failures +✓ spec/lab2-history-index +✓ spec/lab2-persistent-storage +✓ spec/lab2-session-cache +✓ spec/lab3-failover-operations +✓ spec/lab3-idempotent-messages +✓ spec/lab3-load-balancing +✓ spec/lab3-scaling-cost +✓ change/verify-lab3-grade4-recovery +Totals: 16 passed, 0 failed (16 items) +PASS: normative files and pipeline unchanged; no tracked secrets or volumes + +exit: 0 elapsed_seconds: 1.683 diff --git a/evidence/grade4-detection-proxy-interrupted.txt b/evidence/grade4-detection-proxy-interrupted.txt new file mode 100644 index 00000000..63a004f3 --- /dev/null +++ b/evidence/grade4-detection-proxy-interrupted.txt @@ -0,0 +1,65 @@ +date: 2026-11-05T19:16:10.092594+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ make detection +rm -rf evidence/detection +for run in 1 2 3; do sh scripts/detection.sh evidence/detection/run$run || exit 1; done + Container mtusi2026lab3-grade4-20261105-api-1 Running + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-1 Starting + Container mtusi2026lab3-grade4-20261105-api-1 Started + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +{ + "action": [ + "docker", + "stop", + "-t", + "7", + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" + ], + "action_exit": 0, + "action_seconds": 2.761, + "lb_up_before": 0, + "lb_samples_missing": 110, + "lb_up_counted_as_detected": -1, + "detection_seconds": null, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 0, + "x_instance_id_present": false, + "error_code": null + }, + "restore": "scripts/restore.sh", + "restore_seconds": 2.3, + "lb_up_after_restore": null, + "checks": { + "action_ok": true, + "lb_observed": false, + "detected_in_time": false, + "lb_answer_ok": false, + "excluded_before_action_end": false, + "restored": true + }, + "pass": false +} +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-1 Starting + Container mtusi2026lab3-grade4-20261105-api-1 Started + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +make: *** [detection] Terminated: 15 + +exit: 2 elapsed_seconds: 48.801 diff --git a/evidence/grade4-detection-proxy-interrupted/run1/sigterm.json b/evidence/grade4-detection-proxy-interrupted/run1/sigterm.json new file mode 100644 index 00000000..bf6145f7 --- /dev/null +++ b/evidence/grade4-detection-proxy-interrupted/run1/sigterm.json @@ -0,0 +1,35 @@ +{"date": "2026-11-05T19:16:10.092594+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", + "action": [ + "docker", + "stop", + "-t", + "7", + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" + ], + "action_exit": 0, + "action_seconds": 2.761, + "lb_up_before": 0, + "lb_samples_missing": 110, + "lb_up_counted_as_detected": -1, + "detection_seconds": null, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 0, + "x_instance_id_present": false, + "error_code": null + }, + "restore": "scripts/restore.sh", + "restore_seconds": 2.3, + "lb_up_after_restore": null, + "checks": { + "action_ok": true, + "lb_observed": false, + "detected_in_time": false, + "lb_answer_ok": false, + "excluded_before_action_end": false, + "restored": true + }, + "pass": false +} diff --git a/evidence/grade4-detection.txt b/evidence/grade4-detection.txt new file mode 100644 index 00000000..94608420 --- /dev/null +++ b/evidence/grade4-detection.txt @@ -0,0 +1,478 @@ +date: 2026-11-05T19:17:01.673107+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ make detection +rm -rf evidence/detection +for run in 1 2 3; do sh scripts/detection.sh evidence/detection/run$run || exit 1; done + Container mtusi2026lab3-grade4-20261105-api-1 Running + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-1 Starting + Container mtusi2026lab3-grade4-20261105-api-1 Started + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +{ + "action": [ + "docker", + "stop", + "-t", + "7", + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" + ], + "action_exit": 0, + "action_seconds": 2.535, + "lb_up_before": 2, + "lb_samples_missing": 0, + "lb_up_counted_as_detected": 1, + "detection_seconds": 1.069, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 200, + "x_instance_id_present": true, + "error_code": null + }, + "restore": "scripts/restore.sh", + "restore_seconds": 1.969, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "lb_observed": true, + "detected_in_time": true, + "lb_answer_ok": true, + "excluded_before_action_end": true, + "restored": true + }, + "pass": true +} +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-1 Starting + Container mtusi2026lab3-grade4-20261105-api-1 Started + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +{ + "action": [ + "docker", + "kill", + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" + ], + "action_exit": 0, + "action_seconds": 0.113, + "lb_up_before": 2, + "lb_samples_missing": 0, + "lb_up_counted_as_detected": 1, + "detection_seconds": 0.934, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 200, + "x_instance_id_present": true, + "error_code": null + }, + "restore": "scripts/restore.sh", + "restore_seconds": 2.57, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "lb_observed": true, + "detected_in_time": true, + "lb_answer_ok": true, + "restored": true + }, + "pass": true +} +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 +{ + "action": [ + "docker", + "pause", + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" + ], + "action_exit": 0, + "action_seconds": 0.057, + "lb_up_before": 2, + "lb_samples_missing": 0, + "lb_up_counted_as_detected": 1, + "detection_seconds": 3.215, + "declared_max_seconds": 3.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 200, + "x_instance_id_present": true, + "error_code": null + }, + "restore": "docker unpause 7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 && scripts/lb-wait.sh", + "restore_seconds": 2.425, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "lb_observed": true, + "detected_in_time": true, + "lb_answer_ok": true, + "restored": true + }, + "pass": true +} +f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a +f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a +{ + "action": [ + "docker", + "stop", + "f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a" + ], + "action_exit": 0, + "action_seconds": 0.167, + "lb_up_before": 2, + "lb_samples_missing": 0, + "lb_up_counted_as_detected": 0, + "detection_seconds": 1.196, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 503, + "x_instance_id_present": false, + "error_code": "unavailable" + }, + "restore": "docker start f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a && scripts/lb-wait.sh", + "restore_seconds": 2.475, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "lb_observed": true, + "detected_in_time": true, + "lb_answer_ok": true, + "restored": true + }, + "pass": true +} + Container mtusi2026lab3-grade4-20261105-api-1 Running + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-1 Starting + Container mtusi2026lab3-grade4-20261105-api-1 Started + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +{ + "action": [ + "docker", + "stop", + "-t", + "7", + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" + ], + "action_exit": 0, + "action_seconds": 2.662, + "lb_up_before": 2, + "lb_samples_missing": 0, + "lb_up_counted_as_detected": 1, + "detection_seconds": 1.205, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 200, + "x_instance_id_present": true, + "error_code": null + }, + "restore": "scripts/restore.sh", + "restore_seconds": 1.953, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "lb_observed": true, + "detected_in_time": true, + "lb_answer_ok": true, + "excluded_before_action_end": true, + "restored": true + }, + "pass": true +} +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-1 Starting + Container mtusi2026lab3-grade4-20261105-api-1 Started + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +{ + "action": [ + "docker", + "kill", + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" + ], + "action_exit": 0, + "action_seconds": 0.162, + "lb_up_before": 2, + "lb_samples_missing": 0, + "lb_up_counted_as_detected": 1, + "detection_seconds": 0.901, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 200, + "x_instance_id_present": true, + "error_code": null + }, + "restore": "scripts/restore.sh", + "restore_seconds": 3.086, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "lb_observed": true, + "detected_in_time": true, + "lb_answer_ok": true, + "restored": true + }, + "pass": true +} +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 +{ + "action": [ + "docker", + "pause", + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" + ], + "action_exit": 0, + "action_seconds": 0.063, + "lb_up_before": 2, + "lb_samples_missing": 0, + "lb_up_counted_as_detected": 1, + "detection_seconds": 2.817, + "declared_max_seconds": 3.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 200, + "x_instance_id_present": true, + "error_code": null + }, + "restore": "docker unpause 7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 && scripts/lb-wait.sh", + "restore_seconds": 2.378, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "lb_observed": true, + "detected_in_time": true, + "lb_answer_ok": true, + "restored": true + }, + "pass": true +} +f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a +f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a +{ + "action": [ + "docker", + "stop", + "f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a" + ], + "action_exit": 0, + "action_seconds": 0.17, + "lb_up_before": 2, + "lb_samples_missing": 0, + "lb_up_counted_as_detected": 0, + "detection_seconds": 1.263, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 503, + "x_instance_id_present": false, + "error_code": "unavailable" + }, + "restore": "docker start f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a && scripts/lb-wait.sh", + "restore_seconds": 2.455, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "lb_observed": true, + "detected_in_time": true, + "lb_answer_ok": true, + "restored": true + }, + "pass": true +} + Container mtusi2026lab3-grade4-20261105-api-1 Running + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-1 Starting + Container mtusi2026lab3-grade4-20261105-api-1 Started + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +{ + "action": [ + "docker", + "stop", + "-t", + "7", + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" + ], + "action_exit": 0, + "action_seconds": 2.665, + "lb_up_before": 2, + "lb_samples_missing": 0, + "lb_up_counted_as_detected": 1, + "detection_seconds": 1.266, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 200, + "x_instance_id_present": true, + "error_code": null + }, + "restore": "scripts/restore.sh", + "restore_seconds": 1.984, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "lb_observed": true, + "detected_in_time": true, + "lb_answer_ok": true, + "excluded_before_action_end": true, + "restored": true + }, + "pass": true +} +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-1 Starting + Container mtusi2026lab3-grade4-20261105-api-1 Started + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +{ + "action": [ + "docker", + "kill", + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" + ], + "action_exit": 0, + "action_seconds": 0.163, + "lb_up_before": 2, + "lb_samples_missing": 0, + "lb_up_counted_as_detected": 1, + "detection_seconds": 0.886, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 200, + "x_instance_id_present": true, + "error_code": null + }, + "restore": "scripts/restore.sh", + "restore_seconds": 3.095, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "lb_observed": true, + "detected_in_time": true, + "lb_answer_ok": true, + "restored": true + }, + "pass": true +} +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 +7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 +{ + "action": [ + "docker", + "pause", + "7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170" + ], + "action_exit": 0, + "action_seconds": 0.063, + "lb_up_before": 2, + "lb_samples_missing": 0, + "lb_up_counted_as_detected": 1, + "detection_seconds": 2.778, + "declared_max_seconds": 3.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 200, + "x_instance_id_present": true, + "error_code": null + }, + "restore": "docker unpause 7cfd91ddf780f4c432d3f69358c626eed4986c5841c85eab201e9330976e1170 && scripts/lb-wait.sh", + "restore_seconds": 2.416, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "lb_observed": true, + "detected_in_time": true, + "lb_answer_ok": true, + "restored": true + }, + "pass": true +} +f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a +f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a +{ + "action": [ + "docker", + "stop", + "f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a" + ], + "action_exit": 0, + "action_seconds": 0.163, + "lb_up_before": 2, + "lb_samples_missing": 0, + "lb_up_counted_as_detected": 0, + "detection_seconds": 1.239, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 503, + "x_instance_id_present": false, + "error_code": "unavailable" + }, + "restore": "docker start f225c6cd77bb40f6e598bd65a1b50fa8244700978ee4086700860e0fb076857a && scripts/lb-wait.sh", + "restore_seconds": 2.481, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "lb_observed": true, + "detected_in_time": true, + "lb_answer_ok": true, + "restored": true + }, + "pass": true +} + +exit: 0 elapsed_seconds: 60.401 diff --git a/evidence/grade4-development.txt b/evidence/grade4-development.txt new file mode 100644 index 00000000..78de9332 --- /dev/null +++ b/evidence/grade4-development.txt @@ -0,0 +1,21 @@ +date: 2026-11-05T19:18:40.804026+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +Наблюдения при разработке (сводка вывода инструментов, не сырой лог). +На 8a7e027 с WIP scripts/probe.py и student-tests/unit/test_probe.py: +$ .venv/bin/ruff check scripts/probe.py +exit 1: E501, две строки 101/102 символа; ruff format исправил переносы. +$ .venv/bin/ruff check student-tests/unit/test_probe.py +exit 1: ARG005 у аргументов lambda a/kw; переименованы в _a/_kw. +$ make student-test PYTEST_ARGS='-q student-tests/unit/test_probe.py' +14 passed in 0.01s. +После исправлений, до коммита 252e3e3: +$ make check +PASS. +$ make lint +PASS: ruff check, 108 files already formatted, ty. +$ make student-test +285 passed, 12 deselected in 31.08s. +Context7 /alembic/alembic/1.16 не нашёл документацию. Resolve вернул +/sqlalchemy/alembic/rel_1_16_5; async cookbook прочитан успешно. +Context7 /sqlalchemy/sqlalchemy/2.0 вернул ветку rel_2_0_55. Семантика pool_pre_ping +дополнительно сверена с .venv/lib/python3.14/site-packages/sqlalchemy/pool/base.py +у установленной версии 2.0.45; AsyncEngine.dispose и async_sessionmaker уже используются. diff --git a/evidence/grade4-drain.txt b/evidence/grade4-drain.txt new file mode 100644 index 00000000..36dbffcc --- /dev/null +++ b/evidence/grade4-drain.txt @@ -0,0 +1,25 @@ +date: 2026-11-05T19:16:05.604041+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ make student-test 'PYTEST_ARGS=-v -s student-tests/api/test_drain.py student-tests/unit/test_timeouts.py' +.venv/bin/pytest -v -s student-tests/api/test_drain.py student-tests/unit/test_timeouts.py +============================= test session starts ============================== +platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /.venv/bin/python +cachedir: .pytest_cache +rootdir: +configfile: pyproject.toml +plugins: asyncio-1.4.0, anyio-4.15.1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function +collecting ... collected 3 items + +student-tests/api/test_drain.py::test_draining_replica_is_not_ready_but_alive PASSED +student-tests/api/test_drain.py::test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests окно: ready/live (503, 200), порт закрыт через 1.05 с, выход 2.44 с +запрос, принятый до SIGTERM и дописанный после закрытия порта: HTTP/1.1 201 Created +запрос, тело которого так и не пришло: HTTP/1.1 503 Service Unavailable + content-type application/json, тело {'error': {'code': 'unavailable', 'message': 'Сервис временно недоступен', 'request_id': '696dc801542f4169965fa2cd535d8d1a'}} +PASSED +student-tests/unit/test_timeouts.py::test_timeouts_shrink_inwards_and_shutdown_fits_the_stop_budget обнаружение 1.5 с (зависание 3.5 с), окно 2.0 с, дообработка 3.0 с, остановка 6.0 с, хук 7.0 с, stop_grace_period 8.0 с; LB server 10.0 с > запрос 8.0 с > БД 3.0 с; readiness 0.8 с < check 1.0 с +PASSED + +============================== 3 passed in 3.97s =============================== + +exit: 0 elapsed_seconds: 4.452 diff --git a/evidence/grade4-evidence-audit-fixture-token.txt b/evidence/grade4-evidence-audit-fixture-token.txt new file mode 100644 index 00000000..0b5b2379 --- /dev/null +++ b/evidence/grade4-evidence-audit-fixture-token.txt @@ -0,0 +1,48 @@ +date: 2026-11-05T19:32:13.299806+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ python3 -c 'import csv,io,json,re,subprocess +from pathlib import Path +root=Path('"'"'evidence'"'"') +det=list((root/'"'"'detection'"'"').glob('"'"'*/*.json'"'"')) +assert len(det)==12 +for p in det: + d=json.loads(p.read_text()) + assert d['"'"'pass'"'"'] and all(d['"'"'checks'"'"'].values()) + assert d['"'"'detection_seconds'"'"']<=d['"'"'declared_max_seconds'"'"'] +fail=json.loads((root/'"'"'failover-each/summary.json'"'"').read_text()) +assert len(fail)==2 and len({d['"'"'label'"'"'] for d in fail})==2 +for p in list((root/'"'"'failover-each'"'"').glob('"'"'*/result.json'"'"'))+[root/'"'"'rolling/result.json'"'"',root/'"'"'rolling-from-7fc911c/result.json'"'"']: + d=json.loads(p.read_text()) + assert d['"'"'pass'"'"'] and all(d['"'"'checks'"'"'].values()) + assert d['"'"'k6'"'"']['"'"'failed_ratio'"'"']==d['"'"'k6'"'"']['"'"'dropped_iterations'"'"']==d['"'"'probe'"'"']['"'"'failed_attempts'"'"']==0 + assert d['"'"'history'"'"']['"'"'confirmed'"'"']==d['"'"'history'"'"']['"'"'stored'"'"']==d['"'"'probe'"'"']['"'"'attempts'"'"'] + assert all(v==0 for v in d['"'"'lb_counters'"'"'].values()) + trace=list(csv.DictReader(io.StringIO((p.parent/'"'"'lb_up.csv'"'"').read_text().split('"'"'\n'"'"',1)[1]))) + assert min(int(r['"'"'up'"'"']) for r in trace)==d['"'"'lb_up_min'"'"'] + probes=list(csv.DictReader(io.StringIO((p.parent/'"'"'probe.csv'"'"').read_text().split('"'"'\n'"'"',1)[1]))) + first=next(r for r in probes if float(r['"'"'t_rel_s'"'"'])>=0 and r['"'"'status'"'"']=='"'"'201'"'"') + assert abs(float(first['"'"'t_rel_s'"'"'])+float(first['"'"'latency_ms'"'"'])/1000-d['"'"'probe'"'"']['"'"'first_success_s'"'"'])<0.002 + if '"'"'rolling'"'"' in str(p): + action=(p.parent/'"'"'action.txt'"'"').read_text() + assert action.count('"'"'migrate once before the first replacement'"'"')==1 + assert len(re.findall('"'"'replace .+ image='"'"',action))==2 and d['"'"'lb_up_min'"'"']>=2 + print(p,'"'"'PASS'"'"') +paths=subprocess.check_output(['"'"'git'"'"','"'"'diff'"'"','"'"'--name-only'"'"'],text=True).splitlines()+subprocess.check_output(['"'"'git'"'"','"'"'ls-files'"'"','"'"'--others'"'"','"'"'--exclude-standard'"'"'],text=True).splitlines() +pattern=re.compile(r'"'"'Bearer [A-Za-z0-9_-]{25,}|postgresql(?:\+asyncpg)?://[A-Za-z0-9_-]+:[A-Za-z0-9._~%+-]+@|redis://:[A-Za-z0-9._~%+-]+@|"token":\s*"[A-Za-z0-9_-]{25,}"'"'"') +for name in paths: + p=Path(name) + if p.is_file() and p.suffix in {'"'"'.txt'"'"','"'"'.json'"'"','"'"'.csv'"'"','"'"'.md'"'"'}: + assert not pattern.search(p.read_text()),'"'"'possible credential in '"'"'+name +print('"'"'PASS: 12 detection measurements, 4 load scenarios, raw CSV agreement, one migrate per rolling action, credential-pattern scan'"'"') +' +Traceback (most recent call last): + File "", line 33, in + assert not pattern.search(p.read_text()),'possible credential in '+name + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +AssertionError: possible credential in evidence/grade4-student-tests.txt +evidence/failover-each/7cfd91ddf780/result.json PASS +evidence/failover-each/5bbf259ff136/result.json PASS +evidence/rolling/result.json PASS +evidence/rolling-from-7fc911c/result.json PASS + +exit: 1 elapsed_seconds: 0.082 diff --git a/evidence/grade4-evidence-audit-scan-failed.txt b/evidence/grade4-evidence-audit-scan-failed.txt new file mode 100644 index 00000000..f5ddcce3 --- /dev/null +++ b/evidence/grade4-evidence-audit-scan-failed.txt @@ -0,0 +1,53 @@ +date: 2026-11-05T19:31:22.371897+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ python3 -c 'import csv,io,json,re,subprocess +from pathlib import Path + +root=Path('"'"'evidence'"'"') +files=list((root/'"'"'detection'"'"').glob('"'"'*/*.json'"'"')) +assert len(files)==12 +for path in files: + d=json.loads(path.read_text()) + assert d['"'"'pass'"'"'] and all(d['"'"'checks'"'"'].values()),path + assert d['"'"'detection_seconds'"'"'] <= d['"'"'declared_max_seconds'"'"'],path +fail=json.loads((root/'"'"'failover-each/summary.json'"'"').read_text()) +assert len(fail)==2 and len({d['"'"'label'"'"'] for d in fail})==2 +paths=list((root/'"'"'failover-each'"'"').glob('"'"'*/result.json'"'"'))+[root/'"'"'rolling/result.json'"'"',root/'"'"'rolling-from-7fc911c/result.json'"'"'] +for path in paths: + d=json.loads(path.read_text()) + assert d['"'"'pass'"'"'] and all(d['"'"'checks'"'"'].values()),path + assert d['"'"'k6'"'"']['"'"'failed_ratio'"'"']==0 and d['"'"'k6'"'"']['"'"'dropped_iterations'"'"']==0,path + assert d['"'"'history'"'"']['"'"'confirmed'"'"']==d['"'"'history'"'"']['"'"'stored'"'"']==d['"'"'probe'"'"']['"'"'attempts'"'"'],path + assert d['"'"'probe'"'"']['"'"'failed_attempts'"'"']==0 and all(v==0 for v in d['"'"'lb_counters'"'"'].values()),path + trace=list(csv.DictReader(io.StringIO((path.parent/'"'"'lb_up.csv'"'"').read_text().split('"'"'\n'"'"',1)[1]))) + assert min(int(r['"'"'up'"'"']) for r in trace)==d['"'"'lb_up_min'"'"'],path + probes=list(csv.DictReader(io.StringIO((path.parent/'"'"'probe.csv'"'"').read_text().split('"'"'\n'"'"',1)[1]))) + first=next(r for r in probes if float(r['"'"'t_rel_s'"'"'])>=0 and r['"'"'status'"'"']=='"'"'201'"'"') + completed=float(first['"'"'t_rel_s'"'"'])+float(first['"'"'latency_ms'"'"'])/1000 + assert abs(completed-d['"'"'probe'"'"']['"'"'first_success_s'"'"'])<0.002,path + if '"'"'rolling'"'"' in str(path): + action=(path.parent/'"'"'action.txt'"'"').read_text() + assert action.count('"'"'migrate once before the first replacement'"'"')==1 + assert len(re.findall('"'"'replace .+ image='"'"',action))==2 + assert d['"'"'lb_up_min'"'"']>=2 + print(path,'"'"'PASS'"'"',d['"'"'k6'"'"']['"'"'requests'"'"'],'"'"'requests;'"'"',d['"'"'history'"'"']['"'"'stored'"'"'],'"'"'confirmed messages'"'"') +changed=subprocess.check_output(['"'"'git'"'"','"'"'diff'"'"','"'"'--name-only'"'"'],text=True).splitlines() +untracked=subprocess.check_output(['"'"'git'"'"','"'"'ls-files'"'"','"'"'--others'"'"','"'"'--exclude-standard'"'"'],text=True).splitlines() +pattern=re.compile(r'"'"'Bearer [A-Za-z0-9_-]{25,}|postgresql://[^ /\n]+:[^@ /\n]+@|redis://:[^@ /\n]+@|"token":\s*"[A-Za-z0-9_-]{25,}"'"'"') +for name in changed+untracked: + p=Path(name) + if p.is_file() and p.suffix in {'"'"'.txt'"'"','"'"'.json'"'"','"'"'.csv'"'"','"'"'.md'"'"'}: + assert not pattern.search(p.read_text()),'"'"'possible credential in '"'"'+name +print('"'"'PASS: 12 detection measurements, 4 load scenarios, raw CSV agreement, one migrate per rolling action, credential-pattern scan'"'"') +' +Traceback (most recent call last): + File "", line 38, in + assert not pattern.search(p.read_text()),'possible credential in '+name + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +AssertionError: possible credential in evidence/grade4-evidence-audit.txt +evidence/failover-each/7cfd91ddf780/result.json PASS 6001 requests; 645 confirmed messages +evidence/failover-each/5bbf259ff136/result.json PASS 6001 requests; 642 confirmed messages +evidence/rolling/result.json PASS 9001 requests; 913 confirmed messages +evidence/rolling-from-7fc911c/result.json PASS 9001 requests; 913 confirmed messages + +exit: 1 elapsed_seconds: 0.087 diff --git a/evidence/grade4-evidence-audit.txt b/evidence/grade4-evidence-audit.txt new file mode 100644 index 00000000..245c0986 --- /dev/null +++ b/evidence/grade4-evidence-audit.txt @@ -0,0 +1,43 @@ +date: 2026-11-05T19:32:57.655116+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ python3 -c 'import csv,io,json,re,subprocess +from pathlib import Path +root=Path('"'"'evidence'"'"') +det=list((root/'"'"'detection'"'"').glob('"'"'*/*.json'"'"')) +assert len(det)==12 +for p in det: + d=json.loads(p.read_text()) + assert d['"'"'pass'"'"'] and all(d['"'"'checks'"'"'].values()) and d['"'"'detection_seconds'"'"']<=d['"'"'declared_max_seconds'"'"'] +fail=json.loads((root/'"'"'failover-each/summary.json'"'"').read_text()) +assert len(fail)==2 and len({d['"'"'label'"'"'] for d in fail})==2 +for p in list((root/'"'"'failover-each'"'"').glob('"'"'*/result.json'"'"'))+[root/'"'"'rolling/result.json'"'"',root/'"'"'rolling-from-7fc911c/result.json'"'"']: + d=json.loads(p.read_text()) + assert d['"'"'pass'"'"'] and all(d['"'"'checks'"'"'].values()) + assert d['"'"'k6'"'"']['"'"'failed_ratio'"'"']==d['"'"'k6'"'"']['"'"'dropped_iterations'"'"']==d['"'"'probe'"'"']['"'"'failed_attempts'"'"']==0 + assert d['"'"'history'"'"']['"'"'confirmed'"'"']==d['"'"'history'"'"']['"'"'stored'"'"']==d['"'"'probe'"'"']['"'"'attempts'"'"'] + assert all(v==0 for v in d['"'"'lb_counters'"'"'].values()) + trace=list(csv.DictReader(io.StringIO((p.parent/'"'"'lb_up.csv'"'"').read_text().split('"'"'\n'"'"',1)[1]))) + assert min(int(r['"'"'up'"'"']) for r in trace)==d['"'"'lb_up_min'"'"'] + probes=list(csv.DictReader(io.StringIO((p.parent/'"'"'probe.csv'"'"').read_text().split('"'"'\n'"'"',1)[1]))) + first=next(r for r in probes if float(r['"'"'t_rel_s'"'"'])>=0 and r['"'"'status'"'"']=='"'"'201'"'"') + assert abs(float(first['"'"'t_rel_s'"'"'])+float(first['"'"'latency_ms'"'"'])/1000-d['"'"'probe'"'"']['"'"'first_success_s'"'"'])<0.002 + if '"'"'rolling'"'"' in str(p): + action=(p.parent/'"'"'action.txt'"'"').read_text() + assert action.count('"'"'migrate once before the first replacement'"'"')==1 + assert len(re.findall('"'"'replace .+ image='"'"',action))==2 and d['"'"'lb_up_min'"'"']>=2 + print(p,'"'"'PASS'"'"') +paths=subprocess.check_output(['"'"'git'"'"','"'"'diff'"'"','"'"'--name-only'"'"'],text=True).splitlines()+subprocess.check_output(['"'"'git'"'"','"'"'ls-files'"'"','"'"'--others'"'"','"'"'--exclude-standard'"'"'],text=True).splitlines() +pattern=re.compile(r'"'"'Bearer [A-Za-z0-9_-]{25,}|postgresql(?:\+asyncpg)?://[A-Za-z0-9_-]+:[A-Za-z0-9._~%+-]+@|redis://:[A-Za-z0-9._~%+-]+@|"token":\s*"[A-Za-z0-9_-]{25,}"'"'"') +for name in paths: + p=Path(name) + if p.is_file() and p.suffix in {'"'"'.txt'"'"','"'"'.json'"'"','"'"'.csv'"'"','"'"'.md'"'"'}: + assert not pattern.search(p.read_text()),'"'"'possible credential in '"'"'+name +print('"'"'PASS: 12 detection measurements, 4 load scenarios, raw CSV agreement, one migrate per rolling action, credential-pattern scan'"'"') +' +evidence/failover-each/7cfd91ddf780/result.json PASS +evidence/failover-each/5bbf259ff136/result.json PASS +evidence/rolling/result.json PASS +evidence/rolling-from-7fc911c/result.json PASS +PASS: 12 detection measurements, 4 load scenarios, raw CSV agreement, one migrate per rolling action, credential-pattern scan + +exit: 0 elapsed_seconds: 0.065 diff --git a/evidence/grade4-failover-each.txt b/evidence/grade4-failover-each.txt new file mode 100644 index 00000000..d4d48ad6 --- /dev/null +++ b/evidence/grade4-failover-each.txt @@ -0,0 +1,26 @@ +date: 2026-11-05T19:18:13.470607+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ make failover-each RATE=100 +RATE=100 sh scripts/failover-each.sh evidence/failover-each + Container mtusi2026lab3-grade4-20261105-api-1 Running + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +{"label": "7cfd91ddf780", "action": ["sh", "-c", "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' api && { scripts/stand-usage.sh & sleep 10; wait; } && scripts/restore.sh"], "action_exit": 0, "action_seconds": 15.204, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 1, "lb_samples_missing": 0, "detection_seconds": 0.884, "rejoin_seconds": 14.885, "probe": {"attempts": 645, "failed_attempts": 0, "first_failure_s": null, "first_success_s": 0.023, "recovery_s": 0.0, "instances_before": {"5bbf259ff136": 118, "7cfd91ddf780": 115}, "instances_during_outage": {"5bbf259ff136": 126}, "instances_after": {"5bbf259ff136": 142, "7cfd91ddf780": 132}}, "history": {"confirmed": 645, "stored": 645, "exact_match": true, "unique": true}, "lb_counters": {"wretr": 0, "wredis": 0, "econ": 0, "eresp": 0, "hrsp_5xx": 0}, "k6": {"requests": 6001, "rps": 100.02, "ok_rps": 100.02, "failed_ratio": 0, "dropped_iterations": 0, "p50_ms": 2.97, "p95_ms": 3.86, "p99_ms": 4.47, "exit": 0, "rate": 100}, "checks": {"action_ok": true, "load_errors_ok": true, "load_delivered": true, "lb_observed": true, "write_recovery_ok": true, "history_preserved": true, "detected": true, "target_excluded": true, "rejoined": true}, "pass": true} + Container mtusi2026lab3-grade4-20261105-api-1 Running + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +{"label": "5bbf259ff136", "action": ["sh", "-c", "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' api && { scripts/stand-usage.sh & sleep 10; wait; } && scripts/restore.sh"], "action_exit": 0, "action_seconds": 15.2, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 1, "lb_samples_missing": 0, "detection_seconds": 1.498, "rejoin_seconds": 14.99, "probe": {"attempts": 642, "failed_attempts": 0, "first_failure_s": null, "first_success_s": 0.111, "recovery_s": 0.0, "instances_before": {"7cfd91ddf780": 114, "5bbf259ff136": 119}, "instances_during_outage": {"7cfd91ddf780": 121}, "instances_after": {"7cfd91ddf780": 149, "5bbf259ff136": 124}}, "history": {"confirmed": 642, "stored": 642, "exact_match": true, "unique": true}, "lb_counters": {"wretr": 0, "wredis": 0, "econ": 0, "eresp": 0, "hrsp_5xx": 0}, "k6": {"requests": 6001, "rps": 100.02, "ok_rps": 100.02, "failed_ratio": 0, "dropped_iterations": 0, "p50_ms": 2.98, "p95_ms": 3.88, "p99_ms": 4.53, "exit": 0, "rate": 100}, "checks": {"action_ok": true, "load_errors_ok": true, "load_delivered": true, "lb_observed": true, "write_recovery_ok": true, "history_preserved": true, "detected": true, "target_excluded": true, "rejoined": true}, "pass": true} + Container mtusi2026lab3-grade4-20261105-api-1 Running + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy + +exit: 0 elapsed_seconds: 146.608 diff --git a/evidence/grade4-final-review.txt b/evidence/grade4-final-review.txt new file mode 100644 index 00000000..7d8efe20 --- /dev/null +++ b/evidence/grade4-final-review.txt @@ -0,0 +1,33 @@ +date: 2026-11-05T19:32:13.471531+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ sh -ec 'make check; make validate; make lint; openspec validate --all --strict; git diff --check; test -z "$(git diff ef0b9a3 --name-only -- contracts tests README.md COURSE.md .pipeline)"; test -z "$(git diff 252e3e3 --name-only -- src scripts student-tests compose.yaml Dockerfile pyproject.toml uv.lock alembic)"; test -z "$(docker ps -aq --filter label=com.docker.compose.project=mtusi2026lab3-grade4-20261105)"; echo "PASS: runtime code unchanged since evidence SHA; normative files and pipeline unchanged; stage containers stopped"' +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +.venv/bin/ruff check src student-tests alembic scripts/probe.py scripts/limits.py scripts/scaling.py tools +All checks passed! +.venv/bin/ruff format --check src student-tests alembic scripts/probe.py scripts/limits.py scripts/scaling.py tools +108 files already formatted +.venv/bin/ty check src student-tests alembic +All checks passed! +- Validating... +✓ change/2026-11-05-adopt-sqlalchemy-alembic +✓ spec/lab1-event-publication +✓ spec/lab1-messenger-chats +✓ spec/lab1-messenger-history +✓ spec/lab1-messenger-identity +✓ spec/lab1-service-operability +✓ spec/lab2-auth-sessions +✓ spec/lab2-dependency-failures +✓ spec/lab2-history-index +✓ spec/lab2-persistent-storage +✓ spec/lab2-session-cache +✓ spec/lab3-failover-operations +✓ spec/lab3-idempotent-messages +✓ spec/lab3-load-balancing +✓ spec/lab3-scaling-cost +Totals: 15 passed, 0 failed (15 items) +PASS: runtime code unchanged since evidence SHA; normative files and pipeline unchanged; stage containers stopped + +exit: 0 elapsed_seconds: 1.467 diff --git a/evidence/grade4-limits-compose-rejection.txt b/evidence/grade4-limits-compose-rejection.txt new file mode 100644 index 00000000..8fd89f3f --- /dev/null +++ b/evidence/grade4-limits-compose-rejection.txt @@ -0,0 +1,6 @@ +date: 2026-11-05T19:23:22.909799+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ env COMPOSE_FILE=compose.yaml:/tmp/lab3-grade4-violations.yaml docker compose config --quiet +services.deploy.replicas: can't set container_name and api as container name must be unique: invalid compose project + +exit: 1 elapsed_seconds: 0.072 diff --git a/evidence/grade4-limits-negative-config-failed.txt b/evidence/grade4-limits-negative-config-failed.txt new file mode 100644 index 00000000..bc526120 --- /dev/null +++ b/evidence/grade4-limits-negative-config-failed.txt @@ -0,0 +1,20 @@ +date: 2026-11-05T19:19:18.359240+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ env COMPOSE_FILE=compose.yaml:/tmp/lab3-grade4-violations.yaml python3 scripts/limits.py --out evidence/limits-violations.txt +Traceback (most recent call last): + File "/scripts/limits.py", line 133, in + sys.exit(main()) + ~~~~^^ + File "/scripts/limits.py", line 103, in main + config = json.loads(raw) + File "/opt/homebrew/Cellar/python@3.15/3.15.2/Frameworks/Python.framework/Versions/3.15/lib/python3.15/json/__init__.py", line 371, in loads + return _default_decoder.decode(s) + ~~~~~~~~~~~~~~~~~~~~~~~^^^ + File "/opt/homebrew/Cellar/python@3.15/3.15.2/Frameworks/Python.framework/Versions/3.15/lib/python3.15/json/decoder.py", line 358, in decode + obj, end = self.raw_decode(s, idx=_w(s, 0).end()) + ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/homebrew/Cellar/python@3.15/3.15.2/Frameworks/Python.framework/Versions/3.15/lib/python3.15/json/decoder.py", line 376, in raw_decode + raise JSONDecodeError("Expecting value", s, err.value) from None +json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) + +exit: 1 elapsed_seconds: 0.111 diff --git a/evidence/grade4-limits-negative.txt b/evidence/grade4-limits-negative.txt new file mode 100644 index 00000000..e68c8d68 --- /dev/null +++ b/evidence/grade4-limits-negative.txt @@ -0,0 +1,45 @@ +date: 2026-11-05T19:19:26.962774+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ env API_REPLICAS=1 COMPOSE_FILE=compose.yaml:/tmp/lab3-grade4-violations.yaml python3 scripts/limits.py --out evidence/limits-violations.txt +date: 2026-11-05T19:19:27Z commit: 252e3e3 +$ docker compose --profile bench config --format json (declared) +service cpus/mem ports secrets volumes networks ro +-------- ---------- ------------------------------- ------------------------------------------------------ ---------- ---------------------- ----- +api -/512MiB 127.0.0.1:23998 cache_url,cursor_secret,database_url,postgres_password - backend(internal) false +k6 2/512MiB - - - edge false +lb 1/128MiB 127.0.0.1:18083,127.0.0.1:23083 - - backend(internal),edge true +migrate 0.5/256MiB - database_url - backend(internal) false +postgres 2/1024MiB - postgres_password pgdata backend(internal) false +valkey 0.5/128MiB - valkey.conf valkeydata backend(internal) false + +$ docker inspect (running containers: hostname = X-Instance-Id, applied limits) +service hostname cpus memory published +-------- ------------ ---- ------- --------------------------------------------------- +api 5bbf259ff136 1 512MiB +api 7cfd91ddf780 1 512MiB +lb 4f83577b3411 1 128MiB 127.0.0.1:18083->8080/tcp,127.0.0.1:23083->8404/tcp +postgres f225c6cd77bb 2 1024MiB +valkey 5e3ed0a9a35a 0.5 128MiB + +FAIL: api: no CPU or memory limit +FAIL: api: container_name prevents scaling +FAIL: api: secrets ['cache_url', 'cursor_secret', 'database_url', 'postgres_password'], policy ['cache_url', 'cursor_secret', 'database_url'] +FAIL: api: publishes ['127.0.0.1:23998'] in bypass of the balancer +FAIL: api: root filesystem is writable +FAIL: 5 violation(s) + +exit: 1 elapsed_seconds: 0.227 + +Temporary override (configuration was inspected only; never applied): +services: + api: + container_name: forbidden-scale-name + read_only: false + ports: + - "127.0.0.1:23998:8080" + secrets: + - postgres_password + deploy: + resources: + limits: + cpus: "0" diff --git a/evidence/grade4-migration-repeat.txt b/evidence/grade4-migration-repeat.txt new file mode 100644 index 00000000..7eaf1344 --- /dev/null +++ b/evidence/grade4-migration-repeat.txt @@ -0,0 +1,26 @@ +date: 2026-11-05T19:25:09.683166+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ sh -ec ' +migration_counts="SELECT (SELECT version_num FROM alembic_version), (SELECT count(*) FROM users), (SELECT count(*) FROM sessions), (SELECT count(*) FROM chats), (SELECT count(*) FROM chat_members), (SELECT count(*) FROM messages), (SELECT count(*) FROM idempotency_keys)" +migration_before=$(docker compose exec -T postgres psql -X -At -U messenger -d messenger -c "$migration_counts") +echo "before: $migration_before" +make migrate +migration_after=$(docker compose exec -T postgres psql -X -At -U messenger -d messenger -c "$migration_counts") +echo "after: $migration_after" +test "$migration_before" = "$migration_after" +echo "PASS: Alembic head and all table counts unchanged" +docker compose ps -a --format "{{.ID}} {{.Service}} {{.Status}}" +' +before: 0002_idempotency_keys|93|92|46|92|30317|30317 +docker compose run --rm migrate + Container mtusi2026lab3-grade4-20261105-postgres-1 Running +after: 0002_idempotency_keys|93|92|46|92|30317|30317 +PASS: Alembic head and all table counts unchanged +96493b9901ab api Up About a minute (healthy) +dc1e7ee36921 api Up About a minute (healthy) +4f83577b3411 lb Up 9 minutes (healthy) +ad30cd539e85 migrate Exited (0) 9 minutes ago +f225c6cd77bb postgres Up 7 minutes (healthy) +5e3ed0a9a35a valkey Up 9 minutes (healthy) + +exit: 0 elapsed_seconds: 2.273 diff --git a/evidence/grade4-persistence-scoped.txt b/evidence/grade4-persistence-scoped.txt new file mode 100644 index 00000000..4f9681ec --- /dev/null +++ b/evidence/grade4-persistence-scoped.txt @@ -0,0 +1,158 @@ +date: 2026-11-05T19:27:51.938805+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ python3 -c 'import json,subprocess + +def run(cmd): + print('"'"'$'"'"', '"'"' '"'"'.join(cmd), flush=True) + subprocess.run(cmd, check=True) + +def snapshot(label): + ids=subprocess.check_output(['"'"'docker'"'"','"'"'compose'"'"','"'"'ps'"'"','"'"'-aq'"'"'],text=True).split() + raw=json.loads(subprocess.check_output(['"'"'docker'"'"','"'"'inspect'"'"',*ids],text=True)) + states={c['"'"'Id'"'"']: {'"'"'service'"'"':c['"'"'Config'"'"']['"'"'Labels'"'"']['"'"'com.docker.compose.service'"'"'],'"'"'name'"'"':c['"'"'Name'"'"'],'"'"'started'"'"':c['"'"'State'"'"']['"'"'StartedAt'"'"'],'"'"'exit'"'"':c['"'"'State'"'"']['"'"'ExitCode'"'"']} for c in raw} + print(label,json.dumps(states,sort_keys=True),flush=True) + return states + +run(['"'"'make'"'"','"'"'up'"'"']) +before=snapshot('"'"'before'"'"') +run(['"'"'make'"'"','"'"'test-persistence'"'"','"'"'ACTION_SCRIPT=scripts/restart.sh'"'"']) +after=snapshot('"'"'after'"'"') +assert before.keys()==after.keys() +assert len(after)==6 +assert all(after[k]['"'"'started'"'"']!=before[k]['"'"'started'"'"'] for k in before) +assert {d['"'"'service'"'"'] for d in after.values()}=={'"'"'lb'"'"','"'"'api'"'"','"'"'migrate'"'"','"'"'postgres'"'"','"'"'valkey'"'"'} +print('"'"'PASS: all six container IDs preserved, every StartedAt changed'"'"',flush=True) +' +$ make up +sh scripts/secrets.sh +docker compose up -d --build --wait +#1 [internal] load local bake definitions +#1 reading from stdin 448B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 1.65kB done +#2 DONE 0.0s + +#3 [internal] load metadata for ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 +#3 DONE 0.0s + +#4 [internal] load metadata for docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d +#4 DONE 1.5s + +#5 [internal] load .dockerignore +#5 transferring context: 437B done +#5 DONE 0.0s + +#6 FROM ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 +#6 resolve ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 done +#6 DONE 0.0s + +#7 [builder 1/7] FROM docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d +#7 resolve docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d 0.0s done +#7 DONE 0.0s + +#8 [internal] load build context +#8 transferring context: 13.68kB 0.0s done +#8 DONE 0.0s + +#9 [runtime 2/6] RUN groupadd --system --gid 10001 messenger && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger +#9 CACHED + +#10 [builder 6/7] COPY src ./src +#10 CACHED + +#11 [builder 2/7] COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv +#11 CACHED + +#12 [builder 5/7] RUN uv sync --locked --no-dev --no-install-project --no-editable +#12 CACHED + +#13 [builder 7/7] RUN uv sync --locked --no-dev --no-editable +#13 CACHED + +#14 [runtime 4/6] WORKDIR /app +#14 CACHED + +#15 [runtime 3/6] COPY --from=builder --chown=root:root /opt/venv /opt/venv +#15 CACHED + +#16 [builder 3/7] WORKDIR /build +#16 CACHED + +#17 [builder 4/7] COPY pyproject.toml uv.lock README.md ./ +#17 CACHED + +#18 [runtime 5/6] COPY --chown=root:root alembic.ini ./ +#18 CACHED + +#19 [runtime 6/6] COPY --chown=root:root alembic ./alembic +#19 CACHED + +#20 exporting to image +#20 exporting layers done +#20 exporting manifest sha256:5038ad65552ef8a56c20f7f9f042205fa4f08bed2182f7d170da334d08a3bd61 done +#20 exporting config sha256:de74256583c9ad3781e5456b23c7688aa2ee86733f0d16bcd296b9112aee8335 done +#20 exporting attestation manifest sha256:b24ca6722cbe2c1512cb3d26b9933c8cf14d031282d19df5dac611093b428b28 done +#20 exporting manifest list sha256:2a9e9266ba97c082e2a9f89b71fede898269f8459fd53ff355ef0201140b59bb done +#20 naming to docker.io/library/mtusi-messenger:mtusi2026lab3-grade4-20261105 done +#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab3-grade4-20261105 done +#20 DONE 0.1s + +#21 resolving provenance for metadata file +#21 DONE 0.0s + api Built + Network mtusi2026lab3-grade4-20261105_edge Creating + Network mtusi2026lab3-grade4-20261105_edge Created + Network mtusi2026lab3-grade4-20261105_backend Creating + Network mtusi2026lab3-grade4-20261105_backend Created + Container mtusi2026lab3-grade4-20261105-postgres-1 Creating + Container mtusi2026lab3-grade4-20261105-valkey-1 Creating + Container mtusi2026lab3-grade4-20261105-lb-1 Creating + Container mtusi2026lab3-grade4-20261105-valkey-1 Created + Container mtusi2026lab3-grade4-20261105-postgres-1 Created + Container mtusi2026lab3-grade4-20261105-migrate-1 Creating + Container mtusi2026lab3-grade4-20261105-lb-1 Created + Container mtusi2026lab3-grade4-20261105-migrate-1 Created + Container mtusi2026lab3-grade4-20261105-api-2 Creating + Container mtusi2026lab3-grade4-20261105-api-1 Creating + Container mtusi2026lab3-grade4-20261105-api-1 Created + Container mtusi2026lab3-grade4-20261105-api-2 Created + Container mtusi2026lab3-grade4-20261105-postgres-1 Starting + Container mtusi2026lab3-grade4-20261105-lb-1 Starting + Container mtusi2026lab3-grade4-20261105-valkey-1 Starting + Container mtusi2026lab3-grade4-20261105-valkey-1 Started + Container mtusi2026lab3-grade4-20261105-postgres-1 Started + Container mtusi2026lab3-grade4-20261105-postgres-1 Waiting + Container mtusi2026lab3-grade4-20261105-lb-1 Started + Container mtusi2026lab3-grade4-20261105-postgres-1 Healthy + Container mtusi2026lab3-grade4-20261105-migrate-1 Starting + Container mtusi2026lab3-grade4-20261105-migrate-1 Started + Container mtusi2026lab3-grade4-20261105-migrate-1 Waiting + Container mtusi2026lab3-grade4-20261105-migrate-1 Exited + Container mtusi2026lab3-grade4-20261105-api-2 Starting + Container mtusi2026lab3-grade4-20261105-api-2 Started + Container mtusi2026lab3-grade4-20261105-api-1 Starting + Container mtusi2026lab3-grade4-20261105-api-1 Started + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-migrate-1 Waiting + Container mtusi2026lab3-grade4-20261105-postgres-1 Waiting + Container mtusi2026lab3-grade4-20261105-valkey-1 Waiting + Container mtusi2026lab3-grade4-20261105-lb-1 Waiting + Container mtusi2026lab3-grade4-20261105-valkey-1 Healthy + Container mtusi2026lab3-grade4-20261105-migrate-1 Exited + Container mtusi2026lab3-grade4-20261105-postgres-1 Healthy + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy + Container mtusi2026lab3-grade4-20261105-lb-1 Healthy +scripts/lb-wait.sh +before {"223259dd6b9c465577f477f5977c48b2bb35b969b15481cc0e53a1b954e1c778": {"exit": 0, "name": "/mtusi2026lab3-grade4-20261105-migrate-1", "service": "migrate", "started": "2026-11-05T19:27:55.259212054Z"}, "5e25a5e7336b8082474ed3f31c6f649d0b8743fac2104f122aeb4f866ab0bd24": {"exit": 0, "name": "/mtusi2026lab3-grade4-20261105-valkey-1", "service": "valkey", "started": "2026-11-05T19:27:54.684798054Z"}, "a4eb111f811cdeae46efca08b73507c9cefd0d66675a3785f252b55949b3355b": {"exit": 0, "name": "/mtusi2026lab3-grade4-20261105-api-2", "service": "api", "started": "2026-11-05T19:27:56.342284055Z"}, "a5f99d70b71582e67e83392b1d73a57fa44df9002e08c98fd13b77d3da5a04e9": {"exit": 0, "name": "/mtusi2026lab3-grade4-20261105-api-1", "service": "api", "started": "2026-11-05T19:27:56.408012888Z"}, "ca4a75a9bb7163749eeaaa4a5eb64ad29b5cbea663078fe7110179dec807cfc4": {"exit": 0, "name": "/mtusi2026lab3-grade4-20261105-postgres-1", "service": "postgres", "started": "2026-11-05T19:27:54.684199054Z"}, "f850f4461da336beeb05511a783d9bbdae912c8725a7fc893edeb7649becf22f": {"exit": 0, "name": "/mtusi2026lab3-grade4-20261105-lb-1", "service": "lb", "started": "2026-11-05T19:27:54.683165262Z"}} +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18083" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +after {"223259dd6b9c465577f477f5977c48b2bb35b969b15481cc0e53a1b954e1c778": {"exit": 0, "name": "/mtusi2026lab3-grade4-20261105-migrate-1", "service": "migrate", "started": "2026-11-05T19:28:04.468584961Z"}, "5e25a5e7336b8082474ed3f31c6f649d0b8743fac2104f122aeb4f866ab0bd24": {"exit": 0, "name": "/mtusi2026lab3-grade4-20261105-valkey-1", "service": "valkey", "started": "2026-11-05T19:28:03.738892377Z"}, "a4eb111f811cdeae46efca08b73507c9cefd0d66675a3785f252b55949b3355b": {"exit": 0, "name": "/mtusi2026lab3-grade4-20261105-api-2", "service": "api", "started": "2026-11-05T19:28:05.56245167Z"}, "a5f99d70b71582e67e83392b1d73a57fa44df9002e08c98fd13b77d3da5a04e9": {"exit": 0, "name": "/mtusi2026lab3-grade4-20261105-api-1", "service": "api", "started": "2026-11-05T19:28:05.623476253Z"}, "ca4a75a9bb7163749eeaaa4a5eb64ad29b5cbea663078fe7110179dec807cfc4": {"exit": 0, "name": "/mtusi2026lab3-grade4-20261105-postgres-1", "service": "postgres", "started": "2026-11-05T19:28:03.642625252Z"}, "f850f4461da336beeb05511a783d9bbdae912c8725a7fc893edeb7649becf22f": {"exit": 0, "name": "/mtusi2026lab3-grade4-20261105-lb-1", "service": "lb", "started": "2026-11-05T19:28:03.950735002Z"}} +PASS: all six container IDs preserved, every StartedAt changed + +exit: 0 elapsed_seconds: 17.490 diff --git a/evidence/grade4-public-events-unavailable.txt b/evidence/grade4-public-events-unavailable.txt new file mode 100644 index 00000000..f981a35d --- /dev/null +++ b/evidence/grade4-public-events-unavailable.txt @@ -0,0 +1,28 @@ +date: 2026-11-05T19:26:52.960353+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ python3 -c 'import datetime,json,subprocess +from pathlib import Path +line=Path('"'"'evidence/grade4-public.txt'"'"').read_text().splitlines()[0] +since=int(datetime.datetime.fromisoformat(line.split('"'"'date: '"'"')[1].split('"'"' commit:'"'"')[0]).timestamp()) +until=int(datetime.datetime.now(datetime.timezone.utc).timestamp()) +cmd=['"'"'docker'"'"','"'"'events'"'"','"'"'--since'"'"',str(since),'"'"'--until'"'"',str(until),'"'"'--filter'"'"','"'"'type=container'"'"','"'"'--filter'"'"','"'"'label=com.docker.compose.project=mtusi2026lab3-grade4-20261105'"'"','"'"'--format'"'"','"'"'{{json .}}'"'"'] +print('"'"'$'"'"', '"'"' '"'"'.join(cmd)) +events=[json.loads(s) for s in subprocess.check_output(cmd,text=True).splitlines()] +seen={} +for e in events: + if e['"'"'Action'"'"'] in ('"'"'start'"'"','"'"'stop'"'"','"'"'die'"'"','"'"'destroy'"'"'): + attrs=e['"'"'Actor'"'"']['"'"'Attributes'"'"'] + service=attrs.get('"'"'com.docker.compose.service'"'"') + print(e['"'"'time'"'"'],e['"'"'Action'"'"'],service,e['"'"'Actor'"'"']['"'"'ID'"'"'][:12],attrs.get('"'"'name'"'"')) + seen.setdefault(service,set()).add(e['"'"'Action'"'"']) +assert all({'"'"'start'"'"','"'"'die'"'"'} <= seen.get(s,set()) for s in ('"'"'api'"'"','"'"'lb'"'"','"'"'postgres'"'"','"'"'valkey'"'"','"'"'migrate'"'"')),seen +print('"'"'PASS: persistence restarted lb, api, postgres, valkey and migrate; failover target is in grade4-public.txt'"'"') +' +Traceback (most recent call last): + File "", line 16, in + assert all({'start','die'} <= seen.get(s,set()) for s in ('api','lb','postgres','valkey','migrate')),seen + ~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +AssertionError: {} +$ docker events --since 1793906711 --until 1793906812 --filter type=container --filter label=com.docker.compose.project=mtusi2026lab3-grade4-20261105 --format {{json .}} + +exit: 1 elapsed_seconds: 0.077 diff --git a/evidence/grade4-public.txt b/evidence/grade4-public.txt new file mode 100644 index 00000000..84562f2a --- /dev/null +++ b/evidence/grade4-public.txt @@ -0,0 +1,30 @@ +date: 2026-11-05T19:25:11.998157+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ sh -ec 'make test; make test-persistence ACTION_SCRIPT=scripts/restart.sh; make test-failover ACTION_SCRIPT=scripts/stop-one.sh; docker compose ps -a --format "{{.ID}} {{.Service}} {{.Status}}"; make restore; make test' +python3 tests/smoke.py --base-url "http://127.0.0.1:18083" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18083" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18083" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 2.704} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +96493b9901ab api Up 19 seconds (healthy) +dc1e7ee36921 api Exited (0) 12 seconds ago +4f83577b3411 lb Up 20 seconds (healthy) +ad30cd539e85 migrate Exited (0) 19 seconds ago +f225c6cd77bb postgres Up 21 seconds (healthy) +5e3ed0a9a35a valkey Up 21 seconds (healthy) +scripts/restore.sh + Container mtusi2026lab3-grade4-20261105-api-5 Running + Container mtusi2026lab3-grade4-20261105-api-6 Starting + Container mtusi2026lab3-grade4-20261105-api-6 Started + Container mtusi2026lab3-grade4-20261105-api-6 Waiting + Container mtusi2026lab3-grade4-20261105-api-5 Waiting + Container mtusi2026lab3-grade4-20261105-api-6 Healthy + Container mtusi2026lab3-grade4-20261105-api-5 Healthy +python3 tests/smoke.py --base-url "http://127.0.0.1:18083" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. + +exit: 0 elapsed_seconds: 28.513 diff --git a/evidence/grade4-rolling-from-7fc911c.txt b/evidence/grade4-rolling-from-7fc911c.txt new file mode 100644 index 00000000..34926ac4 --- /dev/null +++ b/evidence/grade4-rolling-from-7fc911c.txt @@ -0,0 +1,26 @@ +date: 2026-11-05T19:22:59.139702+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ make rolling-update FROM_REV=7fc911c RATE=100 +git archive "7fc911c" | docker build -q -t "mtusi-messenger:7fc911c" - +sha256:526da3ab8de3b4b90accf804cbdf61eec62cb550d4d74ff82a9b10d30df4bb66 +API_IMAGE="mtusi-messenger:7fc911c" docker compose up -d --no-deps --force-recreate --wait api + Container mtusi2026lab3-grade4-20261105-api-4 Recreate + Container mtusi2026lab3-grade4-20261105-api-3 Recreate + Container mtusi2026lab3-grade4-20261105-api-3 Recreated + Container mtusi2026lab3-grade4-20261105-api-4 Recreated + Container mtusi2026lab3-grade4-20261105-api-4 Starting + Container mtusi2026lab3-grade4-20261105-api-4 Started + Container mtusi2026lab3-grade4-20261105-api-3 Starting + Container mtusi2026lab3-grade4-20261105-api-3 Started + Container mtusi2026lab3-grade4-20261105-api-4 Waiting + Container mtusi2026lab3-grade4-20261105-api-3 Waiting + Container mtusi2026lab3-grade4-20261105-api-4 Healthy + Container mtusi2026lab3-grade4-20261105-api-3 Healthy +scripts/lb-wait.sh +docker compose build -q api +RATE=100 python3 scripts/probe.py load --expect replace --duration 90 \ + --out evidence/rolling-from-7fc911c \ + -- scripts/rolling-update.sh +{"label": "action", "action": ["scripts/rolling-update.sh"], "action_exit": 0, "action_seconds": 13.836, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 2, "lb_samples_missing": 0, "detection_seconds": null, "rejoin_seconds": null, "probe": {"attempts": 913, "failed_attempts": 0, "first_failure_s": null, "first_success_s": 0.082, "recovery_s": 0.0, "instances_before": {"b7b4f1a15dd3": 106, "3b1e610544a0": 127}, "instances_during_outage": {}, "instances_after": {"96493b9901ab": 291, "dc1e7ee36921": 263}}, "history": {"confirmed": 913, "stored": 913, "exact_match": true, "unique": true}, "lb_counters": {"wretr": 0, "wredis": 0, "econ": 0, "eresp": 0, "hrsp_5xx": 0}, "k6": {"requests": 9001, "rps": 100.01, "ok_rps": 100.01, "failed_ratio": 0, "dropped_iterations": 0, "p50_ms": 2.99, "p95_ms": 3.85, "p99_ms": 4.47, "exit": 0, "rate": 100}, "checks": {"action_ok": true, "load_errors_ok": true, "load_delivered": true, "lb_observed": true, "write_recovery_ok": true, "history_preserved": true, "capacity_kept": true, "all_replaced": true}, "pass": true} + +exit: 0 elapsed_seconds: 114.783 diff --git a/evidence/grade4-rolling.txt b/evidence/grade4-rolling.txt new file mode 100644 index 00000000..9f8f7e0c --- /dev/null +++ b/evidence/grade4-rolling.txt @@ -0,0 +1,10 @@ +date: 2026-11-05T19:21:07.986361+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ make rolling-update RATE=100 +docker compose build -q api +RATE=100 python3 scripts/probe.py load --expect replace --duration 90 \ + --out evidence/rolling \ + -- scripts/rolling-update.sh +{"label": "action", "action": ["scripts/rolling-update.sh"], "action_exit": 0, "action_seconds": 10.734, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 2, "lb_samples_missing": 0, "detection_seconds": null, "rejoin_seconds": null, "probe": {"attempts": 913, "failed_attempts": 0, "first_failure_s": null, "first_success_s": 0.055, "recovery_s": 0.0, "instances_before": {"5bbf259ff136": 120, "7cfd91ddf780": 113}, "instances_during_outage": {}, "instances_after": {"a352bb0a17ff": 255, "f6a858d7a20b": 328}}, "history": {"confirmed": 913, "stored": 913, "exact_match": true, "unique": true}, "lb_counters": {"wretr": 0, "wredis": 0, "econ": 0, "eresp": 0, "hrsp_5xx": 0}, "k6": {"requests": 9001, "rps": 100.01, "ok_rps": 100.01, "failed_ratio": 0, "dropped_iterations": 0, "p50_ms": 3.01, "p95_ms": 3.89, "p99_ms": 4.53, "exit": 0, "rate": 100}, "checks": {"action_ok": true, "load_errors_ok": true, "load_delivered": true, "lb_observed": true, "write_recovery_ok": true, "history_preserved": true, "capacity_kept": true, "all_replaced": true}, "pass": true} + +exit: 0 elapsed_seconds: 102.458 diff --git a/evidence/grade4-runtime-limits.txt b/evidence/grade4-runtime-limits.txt new file mode 100644 index 00000000..40fde6e1 --- /dev/null +++ b/evidence/grade4-runtime-limits.txt @@ -0,0 +1,19 @@ +date: 2026-11-05T19:18:40.675349+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ sh -ec 'docker ps -a --filter label=com.docker.compose.project=mtusi2026lab3-grade4-20261105 --format "{{.ID}} {{.Names}} {{.Status}} {{.Ports}}"; docker inspect -f '"'"'{{index .Config.Labels "com.docker.compose.service"}} hostname={{.Config.Hostname}} image={{.Image}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} readonly={{.HostConfig.ReadonlyRootfs}} ports={{json .HostConfig.PortBindings}} user={{.Config.User}}'"'"' $(docker ps -aq --filter label=com.docker.compose.project=mtusi2026lab3-grade4-20261105)' +6ea7ad4475c4 mtusi2026lab3-grade4-20261105-k6-run-66957244d46c Up 25 seconds +5bbf259ff136 mtusi2026lab3-grade4-20261105-api-2 Up 3 minutes (healthy) +7cfd91ddf780 mtusi2026lab3-grade4-20261105-api-1 Up 51 seconds (healthy) +ad30cd539e85 mtusi2026lab3-grade4-20261105-migrate-1 Exited (0) 3 minutes ago +f225c6cd77bb mtusi2026lab3-grade4-20261105-postgres-1 Up 41 seconds (healthy) +4f83577b3411 mtusi2026lab3-grade4-20261105-lb-1 Up 3 minutes (healthy) 127.0.0.1:18083->8080/tcp, 127.0.0.1:23083->8404/tcp +5e3ed0a9a35a mtusi2026lab3-grade4-20261105-valkey-1 Up 3 minutes (healthy) +k6 hostname=6ea7ad4475c4 image=sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8 cpus=2000000000 memory=536870912 readonly=false ports={} user=501:20 +api hostname=5bbf259ff136 image=sha256:fdc0128530c7f092746491b339125eea9fe079b86dcc5c8c3aa89e224fca1706 cpus=1000000000 memory=536870912 readonly=true ports={} user=10001:10001 +api hostname=7cfd91ddf780 image=sha256:fdc0128530c7f092746491b339125eea9fe079b86dcc5c8c3aa89e224fca1706 cpus=1000000000 memory=536870912 readonly=true ports={} user=10001:10001 +migrate hostname=ad30cd539e85 image=sha256:fdc0128530c7f092746491b339125eea9fe079b86dcc5c8c3aa89e224fca1706 cpus=500000000 memory=268435456 readonly=false ports={} user=10001:10001 +postgres hostname=f225c6cd77bb image=sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b cpus=2000000000 memory=1073741824 readonly=false ports={} user=70:70 +lb hostname=4f83577b3411 image=sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b cpus=1000000000 memory=134217728 readonly=true ports={"8080/tcp":[{"HostIp":"127.0.0.1","HostPort":"18083"}],"8404/tcp":[{"HostIp":"127.0.0.1","HostPort":"23083"}]} user=haproxy +valkey hostname=5e3ed0a9a35a image=sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a cpus=500000000 memory=134217728 readonly=false ports={} user=999:1000 + +exit: 0 elapsed_seconds: 0.090 diff --git a/evidence/grade4-start.txt b/evidence/grade4-start.txt new file mode 100644 index 00000000..2bf28f2f --- /dev/null +++ b/evidence/grade4-start.txt @@ -0,0 +1,146 @@ +date: 2026-11-05T19:15:18.997794+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ sh -ec 'docker ps --format "{{.Names}} {{.Status}} {{.Ports}}"; docker volume ls --filter label=com.docker.compose.project=mtusi2026lab3-grade4-20261105; make up; docker compose ps -a; docker compose exec -T api python -c '"'"'from importlib.metadata import version; print({p:version(p) for p in ("sqlalchemy","alembic","gunicorn","uvicorn-worker")})'"'"'; docker compose exec -T postgres psql -X -U messenger -d messenger -c "table alembic_version"' +DRIVER VOLUME NAME +sh scripts/secrets.sh +docker compose up -d --build --wait +#1 [internal] load local bake definitions +#1 reading from stdin 448B done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 1.65kB done +#2 DONE 0.0s + +#3 [internal] load metadata for ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 +#3 DONE 0.0s + +#4 [internal] load metadata for docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d +#4 DONE 1.7s + +#5 [internal] load .dockerignore +#5 transferring context: 437B done +#5 DONE 0.0s + +#6 FROM ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 +#6 resolve ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 0.0s done +#6 DONE 0.0s + +#7 [builder 1/7] FROM docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d +#7 resolve docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d 0.0s done +#7 DONE 0.0s + +#8 [internal] load build context +#8 transferring context: 13.68kB 0.0s done +#8 DONE 0.0s + +#9 [builder 6/7] COPY src ./src +#9 CACHED + +#10 [builder 7/7] RUN uv sync --locked --no-dev --no-editable +#10 CACHED + +#11 [builder 2/7] COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv +#11 CACHED + +#12 [builder 4/7] COPY pyproject.toml uv.lock README.md ./ +#12 CACHED + +#13 [builder 5/7] RUN uv sync --locked --no-dev --no-install-project --no-editable +#13 CACHED + +#14 [runtime 5/6] COPY --chown=root:root alembic.ini ./ +#14 CACHED + +#15 [runtime 3/6] COPY --from=builder --chown=root:root /opt/venv /opt/venv +#15 CACHED + +#16 [runtime 4/6] WORKDIR /app +#16 CACHED + +#17 [runtime 2/6] RUN groupadd --system --gid 10001 messenger && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger +#17 CACHED + +#18 [builder 3/7] WORKDIR /build +#18 CACHED + +#19 [runtime 6/6] COPY --chown=root:root alembic ./alembic +#19 CACHED + +#20 exporting to image +#20 exporting layers done +#20 exporting manifest sha256:5038ad65552ef8a56c20f7f9f042205fa4f08bed2182f7d170da334d08a3bd61 done +#20 exporting config sha256:de74256583c9ad3781e5456b23c7688aa2ee86733f0d16bcd296b9112aee8335 done +#20 exporting attestation manifest sha256:6018cc5e9f5293610ab5cdb4895b36f22c0cfe7ea5b1ac9184dc6b45f6adb3b1 done +#20 exporting manifest list sha256:fdc0128530c7f092746491b339125eea9fe079b86dcc5c8c3aa89e224fca1706 done +#20 naming to docker.io/library/mtusi-messenger:mtusi2026lab3-grade4-20261105 done +#20 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab3-grade4-20261105 done +#20 DONE 0.0s + +#21 resolving provenance for metadata file +#21 DONE 0.0s + api Built + Network mtusi2026lab3-grade4-20261105_edge Creating + Network mtusi2026lab3-grade4-20261105_edge Created + Network mtusi2026lab3-grade4-20261105_backend Creating + Network mtusi2026lab3-grade4-20261105_backend Created + Volume "mtusi2026lab3-grade4-20261105_pgdata" Creating + Volume "mtusi2026lab3-grade4-20261105_pgdata" Created + Volume "mtusi2026lab3-grade4-20261105_valkeydata" Creating + Volume "mtusi2026lab3-grade4-20261105_valkeydata" Created + Container mtusi2026lab3-grade4-20261105-lb-1 Creating + Container mtusi2026lab3-grade4-20261105-postgres-1 Creating + Container mtusi2026lab3-grade4-20261105-valkey-1 Creating + Container mtusi2026lab3-grade4-20261105-valkey-1 Created + Container mtusi2026lab3-grade4-20261105-lb-1 Created + Container mtusi2026lab3-grade4-20261105-postgres-1 Created + Container mtusi2026lab3-grade4-20261105-migrate-1 Creating + Container mtusi2026lab3-grade4-20261105-migrate-1 Created + Container mtusi2026lab3-grade4-20261105-api-2 Creating + Container mtusi2026lab3-grade4-20261105-api-1 Creating + Container mtusi2026lab3-grade4-20261105-api-1 Created + Container mtusi2026lab3-grade4-20261105-api-2 Created + Container mtusi2026lab3-grade4-20261105-postgres-1 Starting + Container mtusi2026lab3-grade4-20261105-valkey-1 Starting + Container mtusi2026lab3-grade4-20261105-lb-1 Starting + Container mtusi2026lab3-grade4-20261105-valkey-1 Started + Container mtusi2026lab3-grade4-20261105-postgres-1 Started + Container mtusi2026lab3-grade4-20261105-postgres-1 Waiting + Container mtusi2026lab3-grade4-20261105-lb-1 Started + Container mtusi2026lab3-grade4-20261105-postgres-1 Healthy + Container mtusi2026lab3-grade4-20261105-migrate-1 Starting + Container mtusi2026lab3-grade4-20261105-migrate-1 Started + Container mtusi2026lab3-grade4-20261105-migrate-1 Waiting + Container mtusi2026lab3-grade4-20261105-migrate-1 Exited + Container mtusi2026lab3-grade4-20261105-api-2 Starting + Container mtusi2026lab3-grade4-20261105-api-2 Started + Container mtusi2026lab3-grade4-20261105-api-1 Starting + Container mtusi2026lab3-grade4-20261105-api-1 Started + Container mtusi2026lab3-grade4-20261105-postgres-1 Waiting + Container mtusi2026lab3-grade4-20261105-valkey-1 Waiting + Container mtusi2026lab3-grade4-20261105-lb-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-migrate-1 Waiting + Container mtusi2026lab3-grade4-20261105-valkey-1 Healthy + Container mtusi2026lab3-grade4-20261105-migrate-1 Exited + Container mtusi2026lab3-grade4-20261105-postgres-1 Healthy + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy + Container mtusi2026lab3-grade4-20261105-lb-1 Healthy +scripts/lb-wait.sh +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab3-grade4-20261105-api-1 mtusi-messenger:mtusi2026lab3-grade4-20261105 "python -m gunicorn …" api 6 seconds ago Up 3 seconds (healthy) +mtusi2026lab3-grade4-20261105-api-2 mtusi-messenger:mtusi2026lab3-grade4-20261105 "python -m gunicorn …" api 6 seconds ago Up 4 seconds (healthy) +mtusi2026lab3-grade4-20261105-lb-1 haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b "docker-entrypoint.s…" lb 6 seconds ago Up 6 seconds (healthy) 127.0.0.1:18083->8080/tcp, 127.0.0.1:23083->8404/tcp +mtusi2026lab3-grade4-20261105-migrate-1 mtusi-messenger:mtusi2026lab3-grade4-20261105 "alembic upgrade head" migrate 6 seconds ago Exited (0) 4 seconds ago +mtusi2026lab3-grade4-20261105-postgres-1 postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b "docker-entrypoint.s…" postgres 6 seconds ago Up 6 seconds (healthy) +mtusi2026lab3-grade4-20261105-valkey-1 valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a "tini -- docker-entr…" valkey 6 seconds ago Up 6 seconds (healthy) +{'sqlalchemy': '2.0.45', 'alembic': '1.16.5', 'gunicorn': '26.2.0', 'uvicorn-worker': '0.4.0'} + version_num +----------------------- + 0002_idempotency_keys +(1 row) + + +exit: 0 elapsed_seconds: 9.454 diff --git a/evidence/grade4-stop-first.txt b/evidence/grade4-stop-first.txt new file mode 100644 index 00000000..a474919c --- /dev/null +++ b/evidence/grade4-stop-first.txt @@ -0,0 +1,37 @@ +date: 2026-11-05T19:26:53.113718+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ sh -ec 'make down; docker ps -a --filter label=com.docker.compose.project=mtusi2026lab3-grade4-20261105 --format "{{.ID}} {{.Names}} {{.Status}}"; docker volume ls --filter label=com.docker.compose.project=mtusi2026lab3-grade4-20261105; test -z "$(docker ps -aq --filter label=com.docker.compose.project=mtusi2026lab3-grade4-20261105)"' +docker compose --profile bench down + Container mtusi2026lab3-grade4-20261105-api-5 Stopping + Container mtusi2026lab3-grade4-20261105-api-6 Stopping + Container mtusi2026lab3-grade4-20261105-lb-1 Stopping + Container mtusi2026lab3-grade4-20261105-lb-1 Stopped + Container mtusi2026lab3-grade4-20261105-lb-1 Removing + Container mtusi2026lab3-grade4-20261105-lb-1 Removed + Container mtusi2026lab3-grade4-20261105-api-6 Stopped + Container mtusi2026lab3-grade4-20261105-api-6 Removing + Container mtusi2026lab3-grade4-20261105-api-6 Removed + Container mtusi2026lab3-grade4-20261105-api-5 Stopped + Container mtusi2026lab3-grade4-20261105-api-5 Removing + Container mtusi2026lab3-grade4-20261105-api-5 Removed + Container mtusi2026lab3-grade4-20261105-valkey-1 Stopping + Container mtusi2026lab3-grade4-20261105-migrate-1 Stopping + Container mtusi2026lab3-grade4-20261105-migrate-1 Stopped + Container mtusi2026lab3-grade4-20261105-migrate-1 Removing + Container mtusi2026lab3-grade4-20261105-migrate-1 Removed + Container mtusi2026lab3-grade4-20261105-postgres-1 Stopping + Container mtusi2026lab3-grade4-20261105-postgres-1 Stopped + Container mtusi2026lab3-grade4-20261105-postgres-1 Removing + Container mtusi2026lab3-grade4-20261105-valkey-1 Stopped + Container mtusi2026lab3-grade4-20261105-valkey-1 Removing + Container mtusi2026lab3-grade4-20261105-postgres-1 Removed + Container mtusi2026lab3-grade4-20261105-valkey-1 Removed + Network mtusi2026lab3-grade4-20261105_backend Removing + Network mtusi2026lab3-grade4-20261105_edge Removing + Network mtusi2026lab3-grade4-20261105_backend Removed + Network mtusi2026lab3-grade4-20261105_edge Removed +DRIVER VOLUME NAME +local mtusi2026lab3-grade4-20261105_pgdata +local mtusi2026lab3-grade4-20261105_valkeydata + +exit: 0 elapsed_seconds: 3.228 diff --git a/evidence/grade4-stop.txt b/evidence/grade4-stop.txt new file mode 100644 index 00000000..4b60be4e --- /dev/null +++ b/evidence/grade4-stop.txt @@ -0,0 +1,37 @@ +date: 2026-11-05T19:29:06.062248+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ sh -ec 'make down; docker ps -a --filter label=com.docker.compose.project=mtusi2026lab3-grade4-20261105 --format "{{.ID}} {{.Names}} {{.Status}}"; docker volume ls --filter label=com.docker.compose.project=mtusi2026lab3-grade4-20261105; test -z "$(docker ps -aq --filter label=com.docker.compose.project=mtusi2026lab3-grade4-20261105)"' +docker compose --profile bench down + Container mtusi2026lab3-grade4-20261105-lb-1 Stopping + Container mtusi2026lab3-grade4-20261105-api-2 Stopping + Container mtusi2026lab3-grade4-20261105-api-1 Stopping + Container mtusi2026lab3-grade4-20261105-lb-1 Stopped + Container mtusi2026lab3-grade4-20261105-lb-1 Removing + Container mtusi2026lab3-grade4-20261105-lb-1 Removed + Container mtusi2026lab3-grade4-20261105-api-1 Stopped + Container mtusi2026lab3-grade4-20261105-api-1 Removing + Container mtusi2026lab3-grade4-20261105-api-2 Stopped + Container mtusi2026lab3-grade4-20261105-api-2 Removing + Container mtusi2026lab3-grade4-20261105-api-1 Removed + Container mtusi2026lab3-grade4-20261105-api-2 Removed + Container mtusi2026lab3-grade4-20261105-valkey-1 Stopping + Container mtusi2026lab3-grade4-20261105-migrate-1 Stopping + Container mtusi2026lab3-grade4-20261105-migrate-1 Stopped + Container mtusi2026lab3-grade4-20261105-migrate-1 Removing + Container mtusi2026lab3-grade4-20261105-migrate-1 Removed + Container mtusi2026lab3-grade4-20261105-postgres-1 Stopping + Container mtusi2026lab3-grade4-20261105-valkey-1 Stopped + Container mtusi2026lab3-grade4-20261105-valkey-1 Removing + Container mtusi2026lab3-grade4-20261105-postgres-1 Stopped + Container mtusi2026lab3-grade4-20261105-postgres-1 Removing + Container mtusi2026lab3-grade4-20261105-valkey-1 Removed + Container mtusi2026lab3-grade4-20261105-postgres-1 Removed + Network mtusi2026lab3-grade4-20261105_backend Removing + Network mtusi2026lab3-grade4-20261105_edge Removing + Network mtusi2026lab3-grade4-20261105_edge Removed + Network mtusi2026lab3-grade4-20261105_backend Removed +DRIVER VOLUME NAME +local mtusi2026lab3-grade4-20261105_pgdata +local mtusi2026lab3-grade4-20261105_valkeydata + +exit: 0 elapsed_seconds: 3.466 diff --git a/evidence/grade4-student-tests.txt b/evidence/grade4-student-tests.txt new file mode 100644 index 00000000..5602bf6d --- /dev/null +++ b/evidence/grade4-student-tests.txt @@ -0,0 +1,303 @@ +date: 2026-11-05T19:26:56.378672+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +NO_PROXY=localhost,127.0.0.1,::1 COMPOSE_PROJECT_NAME=mtusi2026lab3-grade4-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 +$ make student-test PYTEST_ARGS=-vv +.venv/bin/pytest -vv +============================= test session starts ============================== +platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- /.venv/bin/python +cachedir: .pytest_cache +rootdir: +configfile: pyproject.toml +testpaths: student-tests +plugins: asyncio-1.4.0, anyio-4.15.1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function +collecting ... collected 297 items / 12 deselected / 285 selected + +student-tests/api/test_access.py::test_outsider_gets_404_on_chat PASSED [ 0%] +student-tests/api/test_access.py::test_unknown_chat_is_indistinguishable_from_foreign PASSED [ 0%] +student-tests/api/test_access.py::test_outsider_cannot_write PASSED [ 1%] +student-tests/api/test_access.py::test_outsider_cannot_read_history PASSED [ 1%] +student-tests/api/test_access.py::test_access_checked_before_body PASSED [ 1%] +student-tests/api/test_access.py::test_chat_list_hides_foreign_chats PASSED [ 2%] +student-tests/api/test_access.py::test_403_is_never_used PASSED [ 2%] +student-tests/api/test_access.py::test_request_shape_is_checked_before_access PASSED [ 2%] +student-tests/api/test_access.py::test_outsider_does_not_wait_for_members_lock PASSED [ 3%] +student-tests/api/test_bounded_contexts.py::test_identity_does_not_know_about_messaging PASSED [ 3%] +student-tests/api/test_bounded_contexts.py::test_domains_do_not_import_infrastructure PASSED [ 3%] +student-tests/api/test_bounded_contexts.py::test_core_layers_do_not_import_pydantic PASSED [ 4%] +student-tests/api/test_bounded_contexts.py::test_shared_knows_nothing_about_zones PASSED [ 4%] +student-tests/api/test_bounded_contexts.py::test_shared_core_is_free_of_transport PASSED [ 4%] +student-tests/api/test_bounded_contexts.py::test_shared_core_holds_only_platform PASSED [ 5%] +student-tests/api/test_bounded_contexts.py::test_shared_entrypoints_hold_only_transport PASSED [ 5%] +student-tests/api/test_bounded_contexts.py::test_every_zone_owns_its_errors_events_and_commands PASSED [ 5%] +student-tests/api/test_bounded_contexts.py::test_every_zone_owns_its_http_layer PASSED [ 6%] +student-tests/api/test_bounded_contexts.py::test_only_main_knows_both_zones PASSED [ 6%] +student-tests/api/test_bounded_contexts.py::test_application_has_no_separate_entrypoints_root PASSED [ 7%] +student-tests/api/test_bounded_contexts.py::test_shared_source_mentions_no_domain_vocabulary PASSED [ 7%] +student-tests/api/test_bounded_contexts.py::test_shared_declares_no_concrete_commands_or_events PASSED [ 7%] +student-tests/api/test_bounded_contexts.py::test_shared_has_no_test_doubles PASSED [ 8%] +student-tests/api/test_bounded_contexts.py::test_zone_errors_do_not_leak_transport_codes PASSED [ 8%] +student-tests/api/test_bounded_contexts.py::test_sql_of_each_zone_reads_only_its_tables PASSED [ 8%] +student-tests/api/test_bounded_contexts.py::test_messaging_does_not_import_identity_at_all PASSED [ 9%] +student-tests/api/test_bounded_contexts.py::test_sqlalchemy_stays_in_infrastructure_and_asyncpg_is_only_a_driver PASSED [ 9%] +student-tests/api/test_cache.py::test_second_check_is_served_from_cache PASSED [ 9%] +student-tests/api/test_cache.py::test_logout_writes_tombstone_before_database PASSED [ 10%] +student-tests/api/test_cache.py::test_logout_rejected_by_database_changes_nothing PASSED [ 10%] +student-tests/api/test_cache.py::test_fill_racing_with_logout_does_not_revive_token PASSED [ 10%] +student-tests/api/test_cache.py::test_flushall_loses_nothing PASSED [ 11%] +student-tests/api/test_cache.py::test_unreachable_cache_is_bypassed_without_anonymous_access PASSED [ 11%] +student-tests/api/test_concurrency.py::test_hundred_messages_from_ten_clients PASSED [ 11%] +student-tests/api/test_concurrency.py::test_history_follows_the_clock_but_never_moves_back PASSED [ 12%] +student-tests/api/test_concurrency.py::test_parallel_registration_keeps_username_unique PASSED [ 12%] +student-tests/api/test_concurrency.py::test_parallel_chat_creation_is_atomic PASSED [ 12%] +student-tests/api/test_consistency.py::test_registration_is_visible_to_chats_immediately PASSED [ 13%] +student-tests/api/test_consistency.py::test_parallel_registrations_are_all_usable_as_members PASSED [ 13%] +student-tests/api/test_consistency.py::test_rejected_registration_leaves_no_trace PASSED [ 14%] +student-tests/api/test_consistency.py::test_failed_chat_creation_rolls_back_both_tables PASSED [ 14%] +student-tests/api/test_consistency.py::test_rejected_message_does_not_enter_history PASSED [ 14%] +student-tests/api/test_consistency.py::test_confirmed_write_is_visible_to_next_read PASSED [ 15%] +student-tests/api/test_consistency.py::test_parallel_writes_across_zones_keep_counts_consistent PASSED [ 15%] +student-tests/api/test_consistency.py::test_chat_membership_survives_parallel_reads_and_writes PASSED [ 15%] +student-tests/api/test_drain.py::test_draining_replica_is_not_ready_but_alive PASSED [ 16%] +student-tests/api/test_drain.py::test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests PASSED [ 16%] +student-tests/api/test_events.py::test_event_has_no_message_text PASSED [ 16%] +student-tests/api/test_events.py::test_every_subscriber_receives_event PASSED [ 17%] +student-tests/api/test_events.py::test_failing_subscriber_does_not_block_others PASSED [ 17%] +student-tests/api/test_events.py::test_queue_overflow_drops_events_without_raising PASSED [ 17%] +student-tests/api/test_events.py::test_publication_does_not_delay_response PASSED [ 18%] +student-tests/api/test_events.py::test_null_logger_hides_everything PASSED [ 18%] +student-tests/api/test_idempotency.py::test_replay_returns_the_same_message PASSED [ 18%] +student-tests/api/test_idempotency.py::test_same_json_in_other_layout_is_a_replay PASSED [ 19%] +student-tests/api/test_idempotency.py::test_other_body_with_same_key_conflicts PASSED [ 19%] +student-tests/api/test_idempotency.py::test_missing_key_is_rejected PASSED [ 20%] +student-tests/api/test_idempotency.py::test_malformed_key_is_rejected[0] PASSED [ 20%] +student-tests/api/test_idempotency.py::test_malformed_key_is_rejected[1] PASSED [ 20%] +student-tests/api/test_idempotency.py::test_malformed_key_is_rejected[2] PASSED [ 21%] +student-tests/api/test_idempotency.py::test_malformed_key_is_rejected[3] PASSED [ 21%] +student-tests/api/test_idempotency.py::test_malformed_key_is_rejected[4] PASSED [ 21%] +student-tests/api/test_idempotency.py::test_malformed_key_is_rejected[5] PASSED [ 22%] +student-tests/api/test_idempotency.py::test_rejected_attempt_does_not_take_the_key PASSED [ 22%] +student-tests/api/test_idempotency.py::test_key_is_scoped_by_user_and_chat PASSED [ 22%] +student-tests/api/test_idempotency.py::test_replay_checks_session_and_access_again PASSED [ 23%] +student-tests/api/test_idempotency.py::test_expired_key_starts_over PASSED [ 23%] +student-tests/api/test_idempotency.py::test_concurrent_retries_through_two_replicas_store_one_message PASSED [ 23%] +student-tests/api/test_instance.py::test_every_response_names_the_replica PASSED [ 24%] +student-tests/api/test_instance.py::test_rejected_body_names_the_replica PASSED [ 24%] +student-tests/api/test_instance.py::test_unhandled_error_names_the_replica PASSED [ 24%] +student-tests/api/test_instance.py::test_replica_defaults_to_container_hostname PASSED [ 25%] +student-tests/api/test_migrations.py::test_migrations_are_ordered_and_versioned PASSED [ 25%] +student-tests/api/test_migrations.py::test_fresh_schema_matches_models PASSED [ 25%] +student-tests/api/test_migrations.py::test_rerun_changes_nothing PASSED [ 26%] +student-tests/api/test_migrations.py::test_legacy_upgrade_preserves_every_row[1] PASSED [ 26%] +student-tests/api/test_migrations.py::test_legacy_upgrade_preserves_every_row[2] PASSED [ 27%] +student-tests/api/test_migrations.py::test_legacy_drift_is_refused_without_stamping[DROP INDEX messages_history_idx] PASSED [ 27%] +student-tests/api/test_migrations.py::test_legacy_drift_is_refused_without_stamping[ALTER TABLE messages DROP CONSTRAINT messages_chat_id_sender_id_fkey] PASSED [ 27%] +student-tests/api/test_migrations.py::test_legacy_drift_is_refused_without_stamping[ALTER TABLE users DROP CONSTRAINT users_username_format] PASSED [ 28%] +student-tests/api/test_migrations.py::test_legacy_drift_is_refused_without_stamping[ALTER TABLE users ALTER COLUMN display_name TYPE varchar] PASSED [ 28%] +student-tests/api/test_migrations.py::test_legacy_drift_is_refused_without_stamping[ALTER TABLE users ALTER COLUMN username TYPE text COLLATE "C"] PASSED [ 28%] +student-tests/api/test_migrations.py::test_legacy_drift_is_refused_without_stamping[ALTER TABLE sessions SET UNLOGGED] PASSED [ 29%] +student-tests/api/test_migrations.py::test_legacy_drift_is_refused_without_stamping[ALTER TABLE sessions ALTER COLUMN expires_at DROP NOT NULL] PASSED [ 29%] +student-tests/api/test_migrations.py::test_legacy_drift_is_refused_without_stamping[UPDATE schema_migrations SET checksum = 'tampered' WHERE version = '0001_init'] PASSED [ 29%] +student-tests/api/test_migrations.py::test_legacy_drift_is_refused_without_stamping[DELETE FROM schema_migrations WHERE version = '0001_init'] PASSED [ 30%] +student-tests/api/test_migrations.py::test_legacy_drift_is_refused_without_stamping[UPDATE schema_migrations SET version = 'unknown' WHERE version = '0002_idempotency_keys'] PASSED [ 30%] +student-tests/api/test_migrations.py::test_legacy_drift_is_refused_without_stamping[DROP TABLE schema_migrations] PASSED [ 30%] +student-tests/api/test_migrations.py::test_unknown_revision_is_refused PASSED [ 31%] +student-tests/api/test_migrations.py::test_simultaneous_migration_processes_serialize[False] PASSED [ 31%] +student-tests/api/test_migrations.py::test_simultaneous_migration_processes_serialize[True] PASSED [ 31%] +student-tests/api/test_migrations.py::test_sender_must_be_member_even_bypassing_api PASSED [ 32%] +student-tests/api/test_migrations.py::test_key_needs_a_member_and_a_message PASSED [ 32%] +student-tests/api/test_migrations.py::test_failed_revision_rolls_back_ddl_and_version PASSED [ 32%] +student-tests/api/test_negative.py::test_malformed_json PASSED [ 33%] +student-tests/api/test_negative.py::test_unexpected_field PASSED [ 33%] +student-tests/api/test_negative.py::test_wrong_types PASSED [ 34%] +student-tests/api/test_negative.py::test_invalid_registration[body0] PASSED [ 34%] +student-tests/api/test_negative.py::test_invalid_registration[body1] PASSED [ 34%] +student-tests/api/test_negative.py::test_invalid_registration[body2] PASSED [ 35%] +student-tests/api/test_negative.py::test_invalid_registration[body3] PASSED [ 35%] +student-tests/api/test_negative.py::test_invalid_registration[body4] PASSED [ 35%] +student-tests/api/test_negative.py::test_invalid_registration[body5] PASSED [ 36%] +student-tests/api/test_negative.py::test_invalid_registration[body6] PASSED [ 36%] +student-tests/api/test_negative.py::test_invalid_registration[body7] PASSED [ 36%] +student-tests/api/test_negative.py::test_invalid_registration[body8] PASSED [ 37%] +student-tests/api/test_negative.py::test_invalid_registration[body9] PASSED [ 37%] +student-tests/api/test_negative.py::test_invalid_registration[body10] PASSED [ 37%] +student-tests/api/test_negative.py::test_duplicate_username PASSED [ 38%] +student-tests/api/test_negative.py::test_missing_identity PASSED [ 38%] +student-tests/api/test_negative.py::test_invalid_bearer[] PASSED [ 38%] +student-tests/api/test_negative.py::test_invalid_bearer[Bearer] PASSED [ 39%] +student-tests/api/test_negative.py::test_invalid_bearer[Bearer ] PASSED [ 39%] +student-tests/api/test_negative.py::test_invalid_bearer[Bearer not a token] PASSED [ 40%] +student-tests/api/test_negative.py::test_invalid_bearer[Bearer ] PASSED [ 40%] +student-tests/api/test_negative.py::test_invalid_bearer[Token abcdef] PASSED [ 40%] +student-tests/api/test_negative.py::test_invalid_bearer[11111111-1111-4111-8111-111111111111] PASSED [ 41%] +student-tests/api/test_negative.py::test_user_id_header_alone_is_ignored PASSED [ 41%] +student-tests/api/test_negative.py::test_identity_checked_before_body PASSED [ 41%] +student-tests/api/test_negative.py::test_empty_message_rejected PASSED [ 42%] +student-tests/api/test_negative.py::test_long_message_rejected PASSED [ 42%] +student-tests/api/test_negative.py::test_sender_id_not_accepted PASSED [ 42%] +student-tests/api/test_negative.py::test_invalid_chat_creation[body0] PASSED [ 43%] +student-tests/api/test_negative.py::test_invalid_chat_creation[body1] PASSED [ 43%] +student-tests/api/test_negative.py::test_invalid_chat_creation[body2] PASSED [ 43%] +student-tests/api/test_negative.py::test_invalid_chat_creation[body3] PASSED [ 44%] +student-tests/api/test_negative.py::test_invalid_chat_creation[body4] PASSED [ 44%] +student-tests/api/test_negative.py::test_invalid_chat_creation[body5] PASSED [ 44%] +student-tests/api/test_negative.py::test_creator_must_be_member PASSED [ 45%] +student-tests/api/test_negative.py::test_unknown_member_rejected PASSED [ 45%] +student-tests/api/test_negative.py::test_bad_chat_id_in_path PASSED [ 45%] +student-tests/api/test_operability.py::test_health_needs_no_identity PASSED [ 46%] +student-tests/api/test_operability.py::test_every_response_carries_request_id PASSED [ 46%] +student-tests/api/test_operability.py::test_client_request_id_is_preserved PASSED [ 47%] +student-tests/api/test_operability.py::test_unknown_public_route_is_404 PASSED [ 47%] +student-tests/api/test_operability.py::test_unknown_protected_route_needs_identity PASSED [ 47%] +student-tests/api/test_operability.py::test_method_not_allowed_is_hidden PASSED [ 48%] +student-tests/api/test_operability.py::test_logs_do_not_contain_message_text PASSED [ 48%] +student-tests/api/test_operability.py::test_log_lines_are_json PASSED [ 48%] +student-tests/api/test_operability.py::test_body_limit_returns_413 PASSED [ 49%] +student-tests/api/test_operability.py::test_chunked_body_is_limited_too PASSED [ 49%] +student-tests/api/test_operability.py::test_ready_turns_unavailable_without_database PASSED [ 49%] +student-tests/api/test_operability.py::test_protected_request_is_503_not_401_without_database PASSED [ 50%] +student-tests/api/test_operability.py::test_state_survives_new_process PASSED [ 50%] +student-tests/api/test_operability.py::test_logs_do_not_contain_credentials PASSED [ 50%] +student-tests/api/test_pagination.py::test_full_walk_has_no_gaps_or_repeats PASSED [ 51%] +student-tests/api/test_pagination.py::test_limit_may_change_between_pages PASSED [ 51%] +student-tests/api/test_pagination.py::test_last_page_closes_history PASSED [ 51%] +student-tests/api/test_pagination.py::test_cursor_is_shared_between_members PASSED [ 52%] +student-tests/api/test_pagination.py::test_cursor_from_another_chat_is_rejected PASSED [ 52%] +student-tests/api/test_pagination.py::test_broken_cursor_is_rejected[garbage] PASSED [ 52%] +student-tests/api/test_pagination.py::test_broken_cursor_is_rejected[1.2.3.4] PASSED [ 53%] +student-tests/api/test_pagination.py::test_broken_cursor_is_rejected[%20] PASSED [ 53%] +student-tests/api/test_pagination.py::test_broken_cursor_is_rejected[] PASSED [ 54%] +student-tests/api/test_pagination.py::test_limit_out_of_range[0] PASSED [ 54%] +student-tests/api/test_pagination.py::test_limit_out_of_range[-1] PASSED [ 54%] +student-tests/api/test_pagination.py::test_limit_out_of_range[101] PASSED [ 55%] +student-tests/api/test_pagination.py::test_limit_out_of_range[abc] PASSED [ 55%] +student-tests/api/test_pagination.py::test_limit_out_of_range[1.5] PASSED [ 55%] +student-tests/api/test_pagination.py::test_limit_out_of_range[] PASSED [ 56%] +student-tests/api/test_pagination.py::test_default_limit_is_50 PASSED [ 56%] +student-tests/api/test_pagination.py::test_unknown_query_parameter_is_ignored PASSED [ 56%] +student-tests/api/test_pagination.py::test_chat_list_pagination PASSED [ 57%] +student-tests/api/test_pool.py::test_exhausted_pool_answers_503_in_bounded_time PASSED [ 57%] +student-tests/api/test_pool.py::test_statement_timeout_is_set_on_connections PASSED [ 57%] +student-tests/api/test_pool.py::test_idle_session_does_not_acquire_connection PASSED [ 58%] +student-tests/api/test_pool.py::test_closed_idle_connection_is_replaced_before_registration PASSED [ 58%] +student-tests/api/test_pool.py::test_readiness_timeout_releases_waiter PASSED [ 58%] +student-tests/api/test_pool.py::test_server_statement_timeout_returns_connection PASSED [ 59%] +student-tests/api/test_pool.py::test_unit_of_work_persists_sessions_and_events_only_on_commit[False] PASSED [ 59%] +student-tests/api/test_pool.py::test_unit_of_work_persists_sessions_and_events_only_on_commit[True] PASSED [ 60%] +student-tests/api/test_pool.py::test_integrity_failure_rolls_back_and_keeps_pool_usable PASSED [ 60%] +student-tests/api/test_pool.py::test_blocked_message_times_out_without_taking_key[database] PASSED [ 60%] +student-tests/api/test_pool.py::test_blocked_message_times_out_without_taking_key[request] PASSED [ 61%] +student-tests/api/test_pool.py::test_message_integrity_failure_rolls_back_receipt_and_timestamp PASSED [ 61%] +student-tests/api/test_sessions.py::test_login_returns_session_for_registered_user PASSED [ 61%] +student-tests/api/test_sessions.py::test_registration_response_hides_password PASSED [ 62%] +student-tests/api/test_sessions.py::test_password_is_stored_as_argon2id PASSED [ 62%] +student-tests/api/test_sessions.py::test_session_stores_only_token_digest PASSED [ 62%] +student-tests/api/test_sessions.py::test_wrong_password_and_unknown_user_look_the_same PASSED [ 63%] +student-tests/api/test_sessions.py::test_login_without_valid_basic[headers0] PASSED [ 63%] +student-tests/api/test_sessions.py::test_login_without_valid_basic[headers1] PASSED [ 63%] +student-tests/api/test_sessions.py::test_login_without_valid_basic[headers2] PASSED [ 64%] +student-tests/api/test_sessions.py::test_login_without_valid_basic[headers3] PASSED [ 64%] +student-tests/api/test_sessions.py::test_password_with_colon_logs_in PASSED [ 64%] +student-tests/api/test_sessions.py::test_login_with_body_is_rejected PASSED [ 65%] +student-tests/api/test_sessions.py::test_basic_on_business_endpoint_gets_bearer_challenge PASSED [ 65%] +student-tests/api/test_sessions.py::test_user_id_header_does_not_override_owner PASSED [ 65%] +student-tests/api/test_sessions.py::test_authentication_precedes_validation PASSED [ 66%] +student-tests/api/test_sessions.py::test_logout_revokes_only_current_session PASSED [ 66%] +student-tests/api/test_sessions.py::test_short_ttl_session_expires PASSED [ 67%] +student-tests/api/test_sessions.py::test_expired_sessions_are_purged_on_next_login PASSED [ 67%] +student-tests/api/test_sessions.py::test_chat_routes_require_bearer PASSED [ 67%] +student-tests/api/test_timeout.py::test_timeout_returns_unavailable_error_body PASSED [ 68%] +student-tests/api/test_timeout.py::test_timeout_does_not_break_following_requests PASSED [ 68%] +student-tests/api/test_timeout.py::test_timeout_is_logged_without_message_text PASSED [ 68%] +student-tests/api/test_timeout.py::test_normal_request_is_not_affected_by_timeout PASSED [ 69%] +student-tests/api/test_timeout.py::test_slow_event_handler_does_not_trigger_timeout PASSED [ 69%] +student-tests/api/test_timeout.py::test_healthcheck_stays_fast PASSED [ 69%] +student-tests/api/test_wire_format.py::test_created_at_is_rfc3339_utc[whole-second] PASSED [ 70%] +student-tests/api/test_wire_format.py::test_created_at_is_rfc3339_utc[microseconds] PASSED [ 70%] +student-tests/unit/test_authentication.py::test_basic_splits_on_first_colon PASSED [ 70%] +student-tests/unit/test_authentication.py::test_basic_decodes_utf8_password PASSED [ 71%] +student-tests/unit/test_authentication.py::test_basic_scheme_is_case_insensitive PASSED [ 71%] +student-tests/unit/test_authentication.py::test_basic_rejects_malformed[] PASSED [ 71%] +student-tests/unit/test_authentication.py::test_basic_rejects_malformed[Basic] PASSED [ 72%] +student-tests/unit/test_authentication.py::test_basic_rejects_malformed[Basic ] PASSED [ 72%] +student-tests/unit/test_authentication.py::test_basic_rejects_malformed[Basic !!!not-base64!!!] PASSED [ 72%] +student-tests/unit/test_authentication.py::test_basic_rejects_malformed[Basic YWxpY2U] PASSED [ 73%] +student-tests/unit/test_authentication.py::test_basic_rejects_malformed[Basic bm8tY29sb24taGVyZQ==] PASSED [ 73%] +student-tests/unit/test_authentication.py::test_basic_rejects_malformed[Basic 0LDQu9C40YHQsDrQv9Cw0YDQvtC70Yw=] PASSED [ 74%] +student-tests/unit/test_authentication.py::test_basic_rejects_malformed[Basic YWxpY2U6//4=] PASSED [ 74%] +student-tests/unit/test_authentication.py::test_basic_rejects_malformed[Bearer YWxpY2U6c2VjcmV0] PASSED [ 74%] +student-tests/unit/test_authentication.py::test_bearer_accepts_token68 PASSED [ 75%] +student-tests/unit/test_authentication.py::test_bearer_rejects_malformed[] PASSED [ 75%] +student-tests/unit/test_authentication.py::test_bearer_rejects_malformed[Bearer] PASSED [ 75%] +student-tests/unit/test_authentication.py::test_bearer_rejects_malformed[Bearer ] PASSED [ 76%] +student-tests/unit/test_authentication.py::test_bearer_rejects_malformed[Bearer a b] PASSED [ 76%] +student-tests/unit/test_authentication.py::test_bearer_rejects_malformed[Basic abc] PASSED [ 76%] +student-tests/unit/test_authentication.py::test_bearer_rejects_malformed[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] PASSED [ 77%] +student-tests/unit/test_authentication.py::test_bearer_rejects_oversized_token PASSED [ 77%] +student-tests/unit/test_authentication.py::test_challenge_depends_only_on_path PASSED [ 77%] +student-tests/unit/test_cursor.py::test_roundtrip_preserves_key PASSED [ 78%] +student-tests/unit/test_cursor.py::test_cursor_is_opaque PASSED [ 78%] +student-tests/unit/test_cursor.py::test_cursor_from_another_scope_is_rejected PASSED [ 78%] +student-tests/unit/test_cursor.py::test_forged_cursor_is_rejected PASSED [ 79%] +student-tests/unit/test_cursor.py::test_broken_cursor_is_rejected[] PASSED [ 79%] +student-tests/unit/test_cursor.py::test_broken_cursor_is_rejected[garbage] PASSED [ 80%] +student-tests/unit/test_cursor.py::test_broken_cursor_is_rejected[1.2.3] PASSED [ 80%] +student-tests/unit/test_cursor.py::test_broken_cursor_is_rejected[2.1.2.3] PASSED [ 80%] +student-tests/unit/test_cursor.py::test_broken_cursor_is_rejected[1.notanumber.ff.AAAA] PASSED [ 81%] +student-tests/unit/test_cursor.py::test_broken_cursor_is_rejected[1.1.zz.AAAA] PASSED [ 81%] +student-tests/unit/test_cursor.py::test_tampered_payload_is_rejected PASSED [ 81%] +student-tests/unit/test_domain.py::test_username_rejects_invalid[ab] PASSED [ 82%] +student-tests/unit/test_domain.py::test_username_rejects_invalid[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 82%] +student-tests/unit/test_domain.py::test_username_rejects_invalid[Alice] PASSED [ 82%] +student-tests/unit/test_domain.py::test_username_rejects_invalid[alice-01] PASSED [ 83%] +student-tests/unit/test_domain.py::test_username_rejects_invalid[\u0430\u043b\u0438\u0441\u0430] PASSED [ 83%] +student-tests/unit/test_domain.py::test_username_rejects_invalid[] PASSED [ 83%] +student-tests/unit/test_domain.py::test_username_rejects_invalid[a b] PASSED [ 84%] +student-tests/unit/test_domain.py::test_username_rejects_invalid[alice\n] PASSED [ 84%] +student-tests/unit/test_domain.py::test_username_rejects_invalid[alice\r\n] PASSED [ 84%] +student-tests/unit/test_domain.py::test_username_accepts_valid[abc] PASSED [ 85%] +student-tests/unit/test_domain.py::test_username_accepts_valid[a_b_9] PASSED [ 85%] +student-tests/unit/test_domain.py::test_username_accepts_valid[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 85%] +student-tests/unit/test_domain.py::test_display_name_rejects_invalid[] PASSED [ 86%] +student-tests/unit/test_domain.py::test_display_name_rejects_invalid[ ] PASSED [ 86%] +student-tests/unit/test_domain.py::test_display_name_rejects_invalid[\t\n] PASSED [ 87%] +student-tests/unit/test_domain.py::test_display_name_rejects_invalid[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] PASSED [ 87%] +student-tests/unit/test_domain.py::test_password_rejects_invalid[] PASSED [ 87%] +student-tests/unit/test_domain.py::test_password_rejects_invalid[ ] PASSED [ 88%] +student-tests/unit/test_domain.py::test_password_rejects_invalid[xxxxxxxxxxx] PASSED [ 88%] +student-tests/unit/test_domain.py::test_password_rejects_invalid[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] PASSED [ 88%] +student-tests/unit/test_domain.py::test_password_rejects_invalid[\t\t\t\t\t\t\t\t\t\t\t\t] PASSED [ 89%] +student-tests/unit/test_domain.py::test_password_counts_code_points[xxxxxxxxxxxx] PASSED [ 89%] +student-tests/unit/test_domain.py::test_password_counts_code_points[\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f\u043f] PASSED [ 89%] +student-tests/unit/test_domain.py::test_password_counts_code_points[\u043f\u0430\u0440\u043e\u043b\u044c:\u0441:\u0434\u0432\u043e\u0435\u0442\u043e\u0447\u0438\u0435\u043c] PASSED [ 90%] +student-tests/unit/test_domain.py::test_password_counts_code_points[\U0001f44b\U0001f44b\U0001f44b\U0001f44b\U0001f44b\U0001f44b\U0001f44b\U0001f44b\U0001f44b\U0001f44b\U0001f44b\U0001f44b] PASSED [ 90%] +student-tests/unit/test_domain.py::test_password_is_not_printed PASSED [ 90%] +student-tests/unit/test_domain.py::test_session_token_is_opaque_and_stored_as_digest PASSED [ 91%] +student-tests/unit/test_domain.py::test_session_tokens_are_unique PASSED [ 91%] +student-tests/unit/test_domain.py::test_message_text_counts_code_points_not_bytes PASSED [ 91%] +student-tests/unit/test_domain.py::test_message_text_rejects_blank PASSED [ 92%] +student-tests/unit/test_domain.py::test_chat_requires_creator_in_members PASSED [ 92%] +student-tests/unit/test_domain.py::test_chat_rejects_duplicate_members PASSED [ 92%] +student-tests/unit/test_domain.py::test_chat_rejects_more_than_hundred_members PASSED [ 93%] +student-tests/unit/test_domain.py::test_receipt_replays_only_the_same_request PASSED [ 93%] +student-tests/unit/test_domain.py::test_posted_message_describes_itself PASSED [ 94%] +student-tests/unit/test_domain.py::test_user_registration_emits_event PASSED [ 94%] +student-tests/unit/test_domain.py::test_clock_is_strictly_increasing PASSED [ 94%] +student-tests/unit/test_probe.py::test_recovery_requires_completed_success[attempts0-None-False] PASSED [ 95%] +student-tests/unit/test_probe.py::test_recovery_requires_completed_success[attempts1-11-False] PASSED [ 95%] +student-tests/unit/test_probe.py::test_recovery_requires_completed_success[attempts2-1.5-True] PASSED [ 95%] +student-tests/unit/test_probe.py::test_recovery_requires_completed_success[attempts3-0-True] PASSED [ 96%] +student-tests/unit/test_probe.py::test_recovery_requires_completed_success[attempts4-0-False] PASSED [ 96%] +student-tests/unit/test_probe.py::test_incomplete_load_fails[requests-0-load_delivered] PASSED [ 96%] +student-tests/unit/test_probe.py::test_incomplete_load_fails[dropped_iterations-1-load_delivered] PASSED [ 97%] +student-tests/unit/test_probe.py::test_missing_lb_sample_fails PASSED [ 97%] +student-tests/unit/test_probe.py::test_full_history_matches_confirmed_bodies[none] PASSED [ 97%] +student-tests/unit/test_probe.py::test_full_history_matches_confirmed_bodies[missing] PASSED [ 98%] +student-tests/unit/test_probe.py::test_full_history_matches_confirmed_bodies[duplicate] PASSED [ 98%] +student-tests/unit/test_probe.py::test_full_history_matches_confirmed_bodies[changed] PASSED [ 98%] +student-tests/unit/test_probe.py::test_full_history_matches_confirmed_bodies[http] PASSED [ 99%] +student-tests/unit/test_probe.py::test_full_history_matches_confirmed_bodies[cycle] PASSED [ 99%] +student-tests/unit/test_timeouts.py::test_timeouts_shrink_inwards_and_shutdown_fits_the_stop_budget PASSED [100%] + +===================== 285 passed, 12 deselected in 31.94s ====================== + +exit: 0 elapsed_seconds: 32.741 diff --git a/evidence/lb-check.json b/evidence/lb-check.json index 91a01013..2bb830a4 100644 --- a/evidence/lb-check.json +++ b/evidence/lb-check.json @@ -1,7 +1,7 @@ -{ +{"date": "2026-11-05T19:16:58.984609+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "instances_one_token": { - "1b4a73f61b07": 20, - "8ffe0c70f4ea": 20 + "7cfd91ddf780": 20, + "5bbf259ff136": 20 }, "same_key": { "rounds": 20, @@ -10,8 +10,8 @@ "201": 160 }, "instances": { - "8ffe0c70f4ea": 80, - "1b4a73f61b07": 80 + "5bbf259ff136": 80, + "7cfd91ddf780": 80 }, "identical_bodies_rounds": 20, "single_copy_rounds": 20, diff --git a/evidence/limits-violations.txt b/evidence/limits-violations.txt index 69121317..bfc023a2 100644 --- a/evidence/limits-violations.txt +++ b/evidence/limits-violations.txt @@ -1,20 +1,26 @@ -date: 2026-10-30T02:46:59Z commit: e049c26 -# намеренные нарушения политики во временном override (не в репозитории): -# services: -# api: -# ports: ["8081:8080"] -# secrets: [database_url, cache_url, cursor_secret, postgres_password] -# valkey: -# container_name: lab3-valkey -# k6: -# deploy: -# resources: -# limits: {cpus: "0", memory: "0"} -$ COMPOSE_FILE=compose.yaml:violations.yaml python3 scripts/limits.py; echo exit=$? # строки FAIL/PASS +date: 2026-11-05T19:19:26.962774+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +$ docker compose --profile bench config --format json (declared) +service cpus/mem ports secrets volumes networks ro +-------- ---------- ------------------------------- ------------------------------------------------------ ---------- ---------------------- ----- +api -/512MiB 127.0.0.1:23998 cache_url,cursor_secret,database_url,postgres_password - backend(internal) false +k6 2/512MiB - - - edge false +lb 1/128MiB 127.0.0.1:18083,127.0.0.1:23083 - - backend(internal),edge true +migrate 0.5/256MiB - database_url - backend(internal) false +postgres 2/1024MiB - postgres_password pgdata backend(internal) false +valkey 0.5/128MiB - valkey.conf valkeydata backend(internal) false + +$ docker inspect (running containers: hostname = X-Instance-Id, applied limits) +service hostname cpus memory published +-------- ------------ ---- ------- --------------------------------------------------- +api 5bbf259ff136 1 512MiB +api 7cfd91ddf780 1 512MiB +lb 4f83577b3411 1 128MiB 127.0.0.1:18083->8080/tcp,127.0.0.1:23083->8404/tcp +postgres f225c6cd77bb 2 1024MiB +valkey 5e3ed0a9a35a 0.5 128MiB + +FAIL: api: no CPU or memory limit +FAIL: api: container_name prevents scaling FAIL: api: secrets ['cache_url', 'cursor_secret', 'database_url', 'postgres_password'], policy ['cache_url', 'cursor_secret', 'database_url'] -FAIL: api: publishes ['0.0.0.0:8081'] in bypass of the balancer -FAIL: api: publishes ['0.0.0.0:8081'] beyond loopback -FAIL: k6: no CPU or memory limit -FAIL: valkey: container_name prevents scaling +FAIL: api: publishes ['127.0.0.1:23998'] in bypass of the balancer +FAIL: api: root filesystem is writable FAIL: 5 violation(s) -exit=1 diff --git a/evidence/limits.txt b/evidence/limits.txt index bb3534af..3ceb664a 100644 --- a/evidence/limits.txt +++ b/evidence/limits.txt @@ -1,4 +1,4 @@ -date: 2026-10-30T02:46:59Z commit: e049c26 +date: 2026-11-05T19:16:58.984609+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 $ docker compose --profile bench config --format json (declared) service cpus/mem ports secrets volumes networks ro -------- ---------- ------------------------------- ------------------------------------ ---------- ---------------------- ----- @@ -12,10 +12,10 @@ valkey 0.5/128MiB - valkey.conf $ docker inspect (running containers: hostname = X-Instance-Id, applied limits) service hostname cpus memory published -------- ------------ ---- ------- --------------------------------------------------- -api fe75c6c37e5a 1 512MiB -api ff81f93e8b0d 1 512MiB -lb 35712870f3c4 1 128MiB 127.0.0.1:18083->8080/tcp,127.0.0.1:23083->8404/tcp -postgres cd7b0d78bc19 2 1024MiB -valkey c9501d6e445a 0.5 128MiB +api 5bbf259ff136 1 512MiB +api 7cfd91ddf780 1 512MiB +lb 4f83577b3411 1 128MiB 127.0.0.1:18083->8080/tcp,127.0.0.1:23083->8404/tcp +postgres f225c6cd77bb 2 1024MiB +valkey 5e3ed0a9a35a 0.5 128MiB PASS: limits, ports, secrets, volumes and networks match the policy diff --git a/evidence/rolling-from-7fc911c/action.txt b/evidence/rolling-from-7fc911c/action.txt new file mode 100644 index 00000000..b852f7a2 --- /dev/null +++ b/evidence/rolling-from-7fc911c/action.txt @@ -0,0 +1,35 @@ +date: 2026-11-05T19:22:59.139702+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +19:23:38 migrate once before the first replacement +19:23:38 database bytes,messages: 20321971,23694 + Container mtusi2026lab3-grade4-20261105-api-3 Running + Container mtusi2026lab3-grade4-20261105-api-4 Running + Container mtusi2026lab3-grade4-20261105-api-5 Creating + Container mtusi2026lab3-grade4-20261105-api-5 Created + Container mtusi2026lab3-grade4-20261105-api-5 Starting + Container mtusi2026lab3-grade4-20261105-api-5 Started + Container mtusi2026lab3-grade4-20261105-api-5 Waiting + Container mtusi2026lab3-grade4-20261105-api-4 Waiting + Container mtusi2026lab3-grade4-20261105-api-3 Waiting + Container mtusi2026lab3-grade4-20261105-api-5 Healthy + Container mtusi2026lab3-grade4-20261105-api-4 Healthy + Container mtusi2026lab3-grade4-20261105-api-3 Healthy +mtusi2026lab3-grade4-20261105-api-3 cpu=10.63% mem=208.8MiB / 512MiB +mtusi2026lab3-grade4-20261105-api-4 cpu=10.60% mem=210.9MiB / 512MiB +mtusi2026lab3-grade4-20261105-lb-1 cpu=2.25% mem=31.14MiB / 128MiB +mtusi2026lab3-grade4-20261105-postgres-1 cpu=10.99% mem=61.38MiB / 1GiB +mtusi2026lab3-grade4-20261105-valkey-1 cpu=0.35% mem=12.97MiB / 128MiB +19:23:43 replace b7b4f1a15dd3 image=mtusi-messenger:7fc911c@526da3ab8de3 -> 96493b9901ab image=mtusi-messenger:mtusi2026lab3-grade4-20261105@2c6f244ba358 + Container mtusi2026lab3-grade4-20261105-api-4 Running + Container mtusi2026lab3-grade4-20261105-api-5 Running + Container mtusi2026lab3-grade4-20261105-api-6 Creating + Container mtusi2026lab3-grade4-20261105-api-6 Created + Container mtusi2026lab3-grade4-20261105-api-6 Starting + Container mtusi2026lab3-grade4-20261105-api-6 Started + Container mtusi2026lab3-grade4-20261105-api-6 Waiting + Container mtusi2026lab3-grade4-20261105-api-5 Waiting + Container mtusi2026lab3-grade4-20261105-api-4 Waiting + Container mtusi2026lab3-grade4-20261105-api-6 Healthy + Container mtusi2026lab3-grade4-20261105-api-5 Healthy + Container mtusi2026lab3-grade4-20261105-api-4 Healthy +19:23:49 replace 3b1e610544a0 image=mtusi-messenger:7fc911c@526da3ab8de3 -> dc1e7ee36921 image=mtusi-messenger:mtusi2026lab3-grade4-20261105@2c6f244ba358 +19:23:52 done: 2 replicas replaced diff --git a/evidence/rolling-from-7fc911c/k6/k6.txt b/evidence/rolling-from-7fc911c/k6/k6.txt new file mode 100644 index 00000000..5569b613 --- /dev/null +++ b/evidence/rolling-from-7fc911c/k6/k6.txt @@ -0,0 +1,3 @@ +date: 2026-11-05T19:22:59.139702+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +time="2026-11-05T19:23:13Z" level=info msg=LOAD-START source=console +measure: 9001 requests, failed ratio 0 diff --git a/evidence/rolling-from-7fc911c/k6/summary.json b/evidence/rolling-from-7fc911c/k6/summary.json new file mode 100644 index 00000000..42c4924f --- /dev/null +++ b/evidence/rolling-from-7fc911c/k6/summary.json @@ -0,0 +1,309 @@ +{"date": "2026-11-05T19:22:59.139702+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", + "root_group": { + "checks": [ + { + "path": "::history 200", + "id": "fe27b3b8edd292c7f2170a6357b087d0", + "passes": 1998, + "fails": 0, + "name": "history 200" + }, + { + "fails": 0, + "name": "message 201", + "path": "::message 201", + "id": "6445aff337e1ade0bc10402fbac79882", + "passes": 8004 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "p(90)", + "p(95)", + "p(99)", + "max" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": false, + "isStdErrTTY": false, + "testRunDurationMs": 100521.118755 + }, + "metrics": { + "http_req_receiving": { + "values": { + "avg": 0.02336042598487845, + "min": 0.007417, + "med": 0.020583, + "p(90)": 0.033334, + "p(95)": 0.040245799999999964, + "p(99)": 0.06649583999999997, + "max": 0.235875 + }, + "type": "trend", + "contains": "time" + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "p(90)": 0, + "p(95)": 0, + "p(99)": 0, + "max": 0.223375, + "avg": 0.0003165705332272185 + } + }, + "http_req_duration{scenario:measure,name:get_history}": { + "type": "trend", + "contains": "time", + "values": { + "max": 51.33625, + "avg": 3.2254752686733554, + "min": 1.773417, + "med": 3.1035415, + "p(90)": 3.7755707000000003, + "p(95)": 4.02803695, + "p(99)": 5.9091073199999995 + } + }, + "http_req_duration": { + "values": { + "max": 54.582583, + "avg": 3.0785167779546287, + "min": 1.562459, + "med": 2.984333, + "p(90)": 3.6166167, + "p(95)": 3.8743877999999996, + "p(99)": 4.926509159999999 + }, + "type": "trend", + "contains": "time" + }, + "http_req_duration{scenario:measure,name:post_message}": { + "values": { + "avg": 3.001155648119888, + "min": 1.721375, + "med": 2.963667, + "p(90)": 3.5613661999999997, + "p(95)": 3.7859501, + "p(99)": 4.348202519999999, + "max": 54.582583 + }, + "type": "trend", + "contains": "time" + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 10052 + } + }, + "http_reqs": { + "contains": "default", + "values": { + "count": 10052, + "rate": 99.99888704481819 + }, + "type": "counter" + }, + "http_reqs{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 9001, + "rate": 89.5433726910474 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 24806079, + "rate": 246774.80023337013 + } + }, + "dropped_iterations": { + "type": "counter", + "contains": "default", + "values": { + "rate": 0, + "count": 0 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 3.0785167779546287, + "min": 1.562459, + "med": 2.984333, + "p(90)": 3.6166167, + "p(95)": 3.8743877999999996, + "p(99)": 4.926509159999999, + "max": 54.582583 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.0023402607441304984, + "min": 0.000583, + "med": 0.001708, + "p(90)": 0.002542, + "p(95)": 0.003, + "p(99)": 0.006167, + "max": 0.356 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 2928753, + "rate": 29135.698411179106 + } + }, + "iteration_duration": { + "contains": "time", + "values": { + "max": 54.655792, + "avg": 3.131208995700864, + "min": 1.663875, + "med": 3.0715624999999998, + "p(90)": 3.7013167, + "p(95)": 3.9450475999999997, + "p(99)": 4.568889909999999 + }, + "type": "trend" + }, + "http_req_waiting": { + "values": { + "avg": 3.0419979192200586, + "min": 1.535209, + "med": 2.949687, + "p(90)": 3.5740369, + "p(95)": 3.82875175, + "p(99)": 4.86732491, + "max": 54.558167 + }, + "type": "trend", + "contains": "time" + }, + "checks": { + "values": { + "fails": 0, + "rate": 1, + "passes": 10002 + }, + "type": "rate", + "contains": "default" + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 20, + "min": 20, + "max": 20 + } + }, + "iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 9001, + "rate": 89.5433726910474 + } + }, + "http_req_duration{scenario:measure}": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 3.611167, + "p(95)": 3.85175, + "p(99)": 4.467, + "max": 54.582583, + "avg": 3.04586505810465, + "min": 1.721375, + "med": 2.987292 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 0, + "min": 0, + "max": 1 + } + }, + "http_req_failed{scenario:measure}": { + "values": { + "passes": 0, + "fails": 9001, + "rate": 0 + }, + "type": "rate", + "contains": "default" + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 10002, + "rate": 99.50147913074726 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "p(90)": 0, + "p(95)": 0, + "p(99)": 0, + "max": 0 + } + }, + "dropped_iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 0, + "rate": 0 + } + }, + "http_req_sending": { + "values": { + "med": 0.011792, + "p(90)": 0.015958, + "p(95)": 0.018917, + "p(99)": 0.03826924999999999, + "max": 0.248042, + "avg": 0.013158432749701675, + "min": 0.002417 + }, + "type": "trend", + "contains": "time" + } + } +} diff --git a/evidence/rolling-from-7fc911c/lb_up.csv b/evidence/rolling-from-7fc911c/lb_up.csv new file mode 100644 index 00000000..9d9eaa29 --- /dev/null +++ b/evidence/rolling-from-7fc911c/lb_up.csv @@ -0,0 +1,936 @@ +# date: 2026-11-05T19:22:59.139702+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +t_rel_s,up +-25.906,2 +-25.804,2 +-25.697,2 +-25.595,2 +-25.492,2 +-25.389,2 +-25.278,2 +-25.17,2 +-25.063,2 +-24.954,2 +-24.842,2 +-24.737,2 +-24.629,2 +-24.521,2 +-24.413,2 +-24.304,2 +-24.192,2 +-24.087,2 +-23.979,2 +-23.877,2 +-23.772,2 +-23.66,2 +-23.55,2 +-23.438,2 +-23.327,2 +-23.22,2 +-23.112,2 +-22.998,2 +-22.887,2 +-22.778,2 +-22.675,2 +-22.569,2 +-22.46,2 +-22.349,2 +-22.242,2 +-22.134,2 +-22.031,2 +-21.92,2 +-21.812,2 +-21.704,2 +-21.592,2 +-21.484,2 +-21.379,2 +-21.268,2 +-21.156,2 +-21.042,2 +-20.937,2 +-20.824,2 +-20.711,2 +-20.606,2 +-20.493,2 +-20.384,2 +-20.279,2 +-20.176,2 +-20.064,2 +-19.958,2 +-19.854,2 +-19.745,2 +-19.636,2 +-19.526,2 +-19.421,2 +-19.316,2 +-19.21,2 +-19.097,2 +-18.986,2 +-18.876,2 +-18.764,2 +-18.653,2 +-18.545,2 +-18.432,2 +-18.325,2 +-18.221,2 +-18.108,2 +-18.004,2 +-17.892,2 +-17.78,2 +-17.669,2 +-17.558,2 +-17.445,2 +-17.337,2 +-17.225,2 +-17.117,2 +-17.011,2 +-16.904,2 +-16.791,2 +-16.681,2 +-16.574,2 +-16.463,2 +-16.35,2 +-16.244,2 +-16.131,2 +-16.021,2 +-15.908,2 +-15.8,2 +-15.688,2 +-15.577,2 +-15.466,2 +-15.363,2 +-15.252,2 +-15.141,2 +-15.032,2 +-14.924,2 +-14.819,2 +-14.708,2 +-14.605,2 +-14.499,2 +-14.394,2 +-14.281,2 +-14.17,2 +-14.059,2 +-13.955,2 +-13.846,2 +-13.741,2 +-13.627,2 +-13.519,2 +-13.406,2 +-13.296,2 +-13.186,2 +-13.077,2 +-12.971,2 +-12.862,2 +-12.754,2 +-12.641,2 +-12.528,2 +-12.416,2 +-12.308,2 +-12.204,2 +-12.095,2 +-11.982,2 +-11.87,2 +-11.758,2 +-11.654,2 +-11.544,2 +-11.437,2 +-11.324,2 +-11.212,2 +-11.098,2 +-10.986,2 +-10.874,2 +-10.766,2 +-10.654,2 +-10.54,2 +-10.429,2 +-10.321,2 +-10.209,2 +-10.096,2 +-9.983,2 +-9.871,2 +-9.76,2 +-9.646,2 +-9.532,2 +-9.423,2 +-9.314,2 +-9.201,2 +-9.093,2 +-8.99,2 +-8.878,2 +-8.769,2 +-8.664,2 +-8.554,2 +-8.442,2 +-8.332,2 +-8.224,2 +-8.113,2 +-8.0,2 +-7.893,2 +-7.787,2 +-7.682,2 +-7.575,2 +-7.463,2 +-7.354,2 +-7.246,2 +-7.143,2 +-7.04,2 +-6.929,2 +-6.819,2 +-6.712,2 +-6.603,2 +-6.494,2 +-6.384,2 +-6.279,2 +-6.175,2 +-6.062,2 +-5.954,2 +-5.845,2 +-5.74,2 +-5.629,2 +-5.526,2 +-5.414,2 +-5.301,2 +-5.197,2 +-5.088,2 +-4.979,2 +-4.877,2 +-4.77,2 +-4.657,2 +-4.546,2 +-4.438,2 +-4.329,2 +-4.227,2 +-4.118,2 +-4.004,2 +-3.892,2 +-3.779,2 +-3.671,2 +-3.56,2 +-3.456,2 +-3.348,2 +-3.235,2 +-3.131,2 +-3.026,2 +-2.913,2 +-2.802,2 +-2.698,2 +-2.585,2 +-2.48,2 +-2.369,2 +-2.257,2 +-2.155,2 +-2.046,2 +-1.938,2 +-1.824,2 +-1.72,2 +-1.61,2 +-1.504,2 +-1.396,2 +-1.288,2 +-1.182,2 +-1.07,2 +-0.958,2 +-0.851,2 +-0.746,2 +-0.639,2 +-0.531,2 +-0.425,2 +-0.312,2 +-0.208,2 +-0.096,2 +0.012,2 +0.117,2 +0.219,2 +0.331,2 +0.439,2 +0.551,2 +0.654,2 +0.764,2 +0.876,2 +0.985,2 +1.097,2 +1.211,2 +1.312,2 +1.415,2 +1.518,2 +1.624,2 +1.729,2 +1.832,2 +1.943,2 +2.055,2 +2.167,2 +2.278,2 +2.387,2 +2.496,2 +2.601,2 +2.703,2 +2.81,2 +2.92,2 +3.032,2 +3.145,2 +3.257,2 +3.363,2 +3.472,2 +3.577,2 +3.684,2 +3.795,2 +3.904,2 +4.014,2 +4.119,2 +4.229,2 +4.341,2 +4.444,2 +4.557,2 +4.661,3 +4.77,3 +4.877,3 +4.988,3 +5.096,3 +5.203,3 +5.307,3 +5.419,3 +5.525,3 +5.639,3 +5.745,3 +5.854,3 +5.962,3 +6.068,3 +6.178,3 +6.285,3 +6.399,2 +6.506,2 +6.618,2 +6.73,2 +6.835,2 +6.942,2 +7.053,2 +7.16,2 +7.272,2 +7.377,2 +7.478,2 +7.588,2 +7.691,2 +7.794,2 +7.896,2 +8.002,2 +8.11,2 +8.221,2 +8.333,2 +8.444,2 +8.557,2 +8.669,2 +8.778,2 +8.887,2 +8.989,2 +9.093,2 +9.197,2 +9.3,2 +9.408,2 +9.52,2 +9.622,2 +9.73,2 +9.838,2 +9.947,2 +10.051,2 +10.154,2 +10.262,2 +10.373,2 +10.482,2 +10.584,2 +10.687,2 +10.795,2 +10.905,3 +11.01,3 +11.122,3 +11.228,3 +11.338,3 +11.445,3 +11.552,3 +11.66,3 +11.762,3 +11.873,3 +11.976,3 +12.079,2 +12.188,2 +12.294,2 +12.4,2 +12.507,2 +12.618,2 +12.72,2 +12.823,2 +12.928,2 +13.039,2 +13.144,2 +13.256,2 +13.37,2 +13.473,2 +13.585,2 +13.697,2 +13.8,2 +13.911,2 +14.015,2 +14.128,2 +14.236,2 +14.345,2 +14.456,2 +14.569,2 +14.675,2 +14.787,2 +14.891,2 +14.995,2 +15.101,2 +15.209,2 +15.321,2 +15.433,2 +15.547,2 +15.654,2 +15.768,2 +15.876,2 +15.992,2 +16.098,2 +16.205,2 +16.316,2 +16.43,2 +16.539,2 +16.643,2 +16.748,2 +16.861,2 +16.97,2 +17.08,2 +17.19,2 +17.296,2 +17.402,2 +17.515,2 +17.629,2 +17.732,2 +17.835,2 +17.943,2 +18.055,2 +18.161,2 +18.267,2 +18.38,2 +18.493,2 +18.597,2 +18.708,2 +18.818,2 +18.93,2 +19.039,2 +19.146,2 +19.249,2 +19.36,2 +19.47,2 +19.575,2 +19.678,2 +19.782,2 +19.894,2 +20.006,2 +20.112,2 +20.22,2 +20.329,2 +20.441,2 +20.545,2 +20.652,2 +20.761,2 +20.876,2 +20.986,2 +21.095,2 +21.206,2 +21.32,2 +21.424,2 +21.528,2 +21.631,2 +21.736,2 +21.845,2 +21.949,2 +22.061,2 +22.172,2 +22.285,2 +22.398,2 +22.506,2 +22.611,2 +22.72,2 +22.824,2 +22.928,2 +23.032,2 +23.135,2 +23.245,2 +23.353,2 +23.459,2 +23.572,2 +23.684,2 +23.794,2 +23.907,2 +24.019,2 +24.128,2 +24.231,2 +24.338,2 +24.443,2 +24.553,2 +24.661,2 +24.767,2 +24.875,2 +24.978,2 +25.086,2 +25.195,2 +25.307,2 +25.409,2 +25.516,2 +25.629,2 +25.741,2 +25.847,2 +25.959,2 +26.061,2 +26.163,2 +26.269,2 +26.377,2 +26.486,2 +26.593,2 +26.697,2 +26.802,2 +26.91,2 +27.021,2 +27.129,2 +27.234,2 +27.344,2 +27.454,2 +27.566,2 +27.678,2 +27.786,2 +27.891,2 +28.003,2 +28.11,2 +28.219,2 +28.329,2 +28.44,2 +28.545,2 +28.651,2 +28.754,2 +28.861,2 +28.969,2 +29.081,2 +29.197,2 +29.304,2 +29.417,2 +29.529,2 +29.635,2 +29.747,2 +29.853,2 +29.967,2 +30.075,2 +30.186,2 +30.289,2 +30.403,2 +30.512,2 +30.618,2 +30.73,2 +30.833,2 +30.944,2 +31.053,2 +31.162,2 +31.277,2 +31.382,2 +31.494,2 +31.609,2 +31.722,2 +31.827,2 +31.94,2 +32.045,2 +32.151,2 +32.261,2 +32.371,2 +32.484,2 +32.594,2 +32.703,2 +32.811,2 +32.923,2 +33.031,2 +33.143,2 +33.253,2 +33.366,2 +33.479,2 +33.586,2 +33.694,2 +33.805,2 +33.917,2 +34.019,2 +34.126,2 +34.236,2 +34.344,2 +34.453,2 +34.564,2 +34.67,2 +34.777,2 +34.888,2 +35.001,2 +35.107,2 +35.218,2 +35.32,2 +35.433,2 +35.545,2 +35.658,2 +35.764,2 +35.876,2 +35.98,2 +36.092,2 +36.205,2 +36.312,2 +36.417,2 +36.527,2 +36.63,2 +36.744,2 +36.852,2 +36.961,2 +37.068,2 +37.181,2 +37.291,2 +37.402,2 +37.515,2 +37.624,2 +37.736,2 +37.847,2 +37.96,2 +38.072,2 +38.183,2 +38.29,2 +38.394,2 +38.502,2 +38.615,2 +38.719,2 +38.831,2 +38.942,2 +39.055,2 +39.161,2 +39.275,2 +39.386,2 +39.496,2 +39.602,2 +39.711,2 +39.825,2 +39.936,2 +40.05,2 +40.163,2 +40.276,2 +40.386,2 +40.494,2 +40.604,2 +40.711,2 +40.819,2 +40.932,2 +41.045,2 +41.158,2 +41.27,2 +41.381,2 +41.489,2 +41.594,2 +41.699,2 +41.802,2 +41.906,2 +42.012,2 +42.12,2 +42.227,2 +42.332,2 +42.444,2 +42.552,2 +42.662,2 +42.767,2 +42.881,2 +42.991,2 +43.105,2 +43.214,2 +43.328,2 +43.44,2 +43.544,2 +43.657,2 +43.769,2 +43.872,2 +43.98,2 +44.086,2 +44.194,2 +44.301,2 +44.41,2 +44.516,2 +44.627,2 +44.735,2 +44.844,2 +44.955,2 +45.068,2 +45.18,2 +45.283,2 +45.394,2 +45.502,2 +45.612,2 +45.72,2 +45.826,2 +45.936,2 +46.045,2 +46.158,2 +46.269,2 +46.381,2 +46.496,2 +46.608,2 +46.719,2 +46.828,2 +46.939,2 +47.044,2 +47.158,2 +47.269,2 +47.376,2 +47.489,2 +47.593,2 +47.701,2 +47.805,2 +47.913,2 +48.023,2 +48.13,2 +48.236,2 +48.347,2 +48.451,2 +48.559,2 +48.672,2 +48.776,2 +48.888,2 +48.997,2 +49.105,2 +49.21,2 +49.324,2 +49.435,2 +49.548,2 +49.66,2 +49.771,2 +49.88,2 +49.983,2 +50.093,2 +50.207,2 +50.315,2 +50.427,2 +50.537,2 +50.649,2 +50.759,2 +50.868,2 +50.971,2 +51.079,2 +51.191,2 +51.298,2 +51.402,2 +51.514,2 +51.627,2 +51.735,2 +51.843,2 +51.953,2 +52.06,2 +52.171,2 +52.281,2 +52.391,2 +52.494,2 +52.606,2 +52.718,2 +52.828,2 +52.939,2 +53.043,2 +53.154,2 +53.258,2 +53.368,2 +53.48,2 +53.584,2 +53.69,2 +53.794,2 +53.903,2 +54.01,2 +54.115,2 +54.22,2 +54.327,2 +54.438,2 +54.551,2 +54.659,2 +54.768,2 +54.875,2 +54.985,2 +55.093,2 +55.202,2 +55.316,2 +55.428,2 +55.535,2 +55.64,2 +55.751,2 +55.862,2 +55.973,2 +56.079,2 +56.187,2 +56.293,2 +56.407,2 +56.51,2 +56.613,2 +56.72,2 +56.834,2 +56.941,2 +57.046,2 +57.152,2 +57.256,2 +57.36,2 +57.468,2 +57.581,2 +57.696,2 +57.804,2 +57.909,2 +58.02,2 +58.132,2 +58.245,2 +58.36,2 +58.468,2 +58.581,2 +58.688,2 +58.795,2 +58.905,2 +59.009,2 +59.118,2 +59.225,2 +59.335,2 +59.441,2 +59.553,2 +59.665,2 +59.776,2 +59.88,2 +59.983,2 +60.093,2 +60.201,2 +60.312,2 +60.426,2 +60.538,2 +60.642,2 +60.755,2 +60.866,2 +60.978,2 +61.09,2 +61.194,2 +61.307,2 +61.415,2 +61.528,2 +61.631,2 +61.742,2 +61.85,2 +61.963,2 +62.075,2 +62.188,2 +62.301,2 +62.41,2 +62.519,2 +62.632,2 +62.739,2 +62.849,2 +62.96,2 +63.069,2 +63.181,2 +63.293,2 +63.4,2 +63.51,2 +63.614,2 +63.726,2 +63.839,2 +63.951,2 +64.064,2 +64.173,2 +64.28,2 +64.384,2 +64.493,2 +64.606,2 +64.717,2 +64.826,2 +64.938,2 +65.045,2 +65.156,2 +65.268,2 +65.38,2 +65.487,2 +65.596,2 +65.702,2 +65.812,2 +65.921,2 +66.035,2 +66.148,2 +66.26,2 +66.368,2 +66.473,2 +66.587,2 +66.7,2 +66.813,2 +66.92,2 +67.028,2 +67.131,2 +67.243,2 +67.351,2 +67.454,2 +67.562,2 +67.674,2 +67.786,2 +67.892,2 +68.004,2 +68.108,2 +68.218,2 +68.325,2 +68.429,2 +68.541,2 +68.654,2 +68.766,2 +68.868,2 +68.975,2 +69.08,2 +69.192,2 +69.295,2 +69.398,2 +69.501,2 +69.607,2 +69.718,2 +69.823,2 +69.93,2 +70.035,2 +70.146,2 +70.258,2 +70.362,2 +70.468,2 +70.579,2 +70.681,2 +70.79,2 +70.896,2 +71.004,2 +71.109,2 +71.22,2 +71.325,2 +71.433,2 +71.545,2 +71.651,2 +71.762,2 +71.876,2 +71.981,2 +72.084,2 +72.195,2 +72.301,2 +72.412,2 +72.522,2 +72.634,2 +72.743,2 +72.847,2 +72.958,2 +73.067,2 +73.176,2 +73.278,2 +73.384,2 +73.493,2 +73.601,2 +73.708,2 +73.817,2 +73.922,2 +74.034,2 +74.149,2 +74.259,2 +74.372,2 +74.479,2 +74.584,2 +74.689,2 +74.801,2 +74.909,2 +75.012,2 +75.121,2 +75.234,2 +75.341,2 +75.453,2 +75.56,2 diff --git a/evidence/rolling-from-7fc911c/probe.csv b/evidence/rolling-from-7fc911c/probe.csv new file mode 100644 index 00000000..e4b464f9 --- /dev/null +++ b/evidence/rolling-from-7fc911c/probe.csv @@ -0,0 +1,915 @@ +# date: 2026-11-05T19:22:59.139702+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +t_rel_s,status,latency_ms,instance +-25.906,201,11.4,b7b4f1a15dd3 +-25.793,201,8.9,3b1e610544a0 +-25.684,201,38.1,b7b4f1a15dd3 +-25.545,201,7.6,3b1e610544a0 +-25.427,201,5.8,b7b4f1a15dd3 +-25.321,201,4.0,3b1e610544a0 +-25.209,201,3.6,b7b4f1a15dd3 +-25.104,201,4.2,3b1e610544a0 +-24.993,201,4.1,3b1e610544a0 +-24.885,201,3.4,3b1e610544a0 +-24.778,201,4.6,3b1e610544a0 +-24.668,201,4.8,3b1e610544a0 +-24.557,201,4.4,3b1e610544a0 +-24.449,201,4.7,3b1e610544a0 +-24.337,201,4.4,3b1e610544a0 +-24.227,201,4.5,3b1e610544a0 +-24.117,201,5.8,3b1e610544a0 +-24.003,201,4.1,b7b4f1a15dd3 +-23.891,201,4.5,b7b4f1a15dd3 +-23.779,201,4.7,b7b4f1a15dd3 +-23.667,201,5.7,b7b4f1a15dd3 +-23.556,201,5.0,b7b4f1a15dd3 +-23.45,201,5.8,b7b4f1a15dd3 +-23.338,201,4.9,b7b4f1a15dd3 +-23.223,201,4.6,3b1e610544a0 +-23.11,201,3.5,3b1e610544a0 +-23.004,201,3.7,3b1e610544a0 +-22.893,201,4.1,3b1e610544a0 +-22.785,201,5.0,b7b4f1a15dd3 +-22.67,201,4.7,3b1e610544a0 +-22.557,201,6.4,3b1e610544a0 +-22.443,201,4.6,b7b4f1a15dd3 +-22.332,201,3.7,b7b4f1a15dd3 +-22.219,201,4.2,b7b4f1a15dd3 +-22.107,201,4.8,b7b4f1a15dd3 +-21.995,201,5.1,b7b4f1a15dd3 +-21.883,201,5.2,3b1e610544a0 +-21.77,201,4.4,3b1e610544a0 +-21.656,201,5.4,3b1e610544a0 +-21.544,201,5.2,b7b4f1a15dd3 +-21.435,201,5.7,3b1e610544a0 +-21.323,201,6.3,b7b4f1a15dd3 +-21.207,201,5.1,b7b4f1a15dd3 +-21.101,201,4.6,b7b4f1a15dd3 +-20.993,201,6.6,b7b4f1a15dd3 +-20.88,201,5.2,b7b4f1a15dd3 +-20.771,201,4.9,b7b4f1a15dd3 +-20.662,201,6.0,b7b4f1a15dd3 +-20.555,201,6.0,3b1e610544a0 +-20.439,201,6.2,b7b4f1a15dd3 +-20.323,201,5.3,3b1e610544a0 +-20.212,201,5.0,3b1e610544a0 +-20.104,201,7.1,3b1e610544a0 +-19.988,201,5.2,3b1e610544a0 +-19.877,201,5.8,3b1e610544a0 +-19.766,201,5.7,3b1e610544a0 +-19.652,201,5.7,b7b4f1a15dd3 +-19.537,201,5.3,b7b4f1a15dd3 +-19.43,201,6.3,b7b4f1a15dd3 +-19.32,201,6.8,b7b4f1a15dd3 +-19.21,201,4.3,b7b4f1a15dd3 +-19.097,201,6.3,b7b4f1a15dd3 +-18.985,201,4.7,b7b4f1a15dd3 +-18.876,201,5.3,b7b4f1a15dd3 +-18.763,201,5.7,3b1e610544a0 +-18.654,201,5.9,3b1e610544a0 +-18.546,201,6.5,3b1e610544a0 +-18.436,201,6.1,b7b4f1a15dd3 +-18.32,201,4.2,3b1e610544a0 +-18.205,201,4.8,3b1e610544a0 +-18.098,201,3.5,3b1e610544a0 +-17.99,201,4.2,3b1e610544a0 +-17.876,201,2.9,3b1e610544a0 +-17.772,201,2.6,3b1e610544a0 +-17.66,201,3.9,3b1e610544a0 +-17.549,201,4.7,3b1e610544a0 +-17.439,201,5.1,3b1e610544a0 +-17.324,201,5.4,b7b4f1a15dd3 +-17.214,201,4.8,b7b4f1a15dd3 +-17.102,201,5.0,b7b4f1a15dd3 +-16.987,201,4.9,b7b4f1a15dd3 +-16.872,201,5.7,3b1e610544a0 +-16.756,201,4.9,3b1e610544a0 +-16.65,201,5.3,3b1e610544a0 +-16.535,201,5.2,3b1e610544a0 +-16.429,201,5.2,3b1e610544a0 +-16.32,201,4.4,3b1e610544a0 +-16.213,201,4.3,3b1e610544a0 +-16.105,201,6.9,b7b4f1a15dd3 +-15.988,201,5.7,3b1e610544a0 +-15.873,201,6.9,b7b4f1a15dd3 +-15.756,201,6.1,b7b4f1a15dd3 +-15.643,201,5.9,3b1e610544a0 +-15.529,201,4.7,3b1e610544a0 +-15.418,201,5.4,3b1e610544a0 +-15.309,201,5.9,3b1e610544a0 +-15.203,201,4.1,3b1e610544a0 +-15.089,201,4.0,3b1e610544a0 +-14.976,201,4.8,b7b4f1a15dd3 +-14.867,201,6.3,b7b4f1a15dd3 +-14.751,201,5.5,3b1e610544a0 +-14.635,201,5.3,3b1e610544a0 +-14.529,201,6.1,3b1e610544a0 +-14.413,201,5.5,3b1e610544a0 +-14.304,201,5.8,3b1e610544a0 +-14.195,201,5.5,3b1e610544a0 +-14.087,201,4.6,3b1e610544a0 +-13.973,201,6.5,b7b4f1a15dd3 +-13.856,201,5.2,b7b4f1a15dd3 +-13.741,201,5.7,3b1e610544a0 +-13.628,201,4.7,3b1e610544a0 +-13.523,201,5.0,b7b4f1a15dd3 +-13.408,201,4.7,3b1e610544a0 +-13.293,201,4.7,3b1e610544a0 +-13.179,201,4.1,b7b4f1a15dd3 +-13.073,201,4.9,3b1e610544a0 +-12.965,201,5.1,3b1e610544a0 +-12.856,201,3.8,3b1e610544a0 +-12.746,201,3.1,3b1e610544a0 +-12.641,201,4.6,3b1e610544a0 +-12.526,201,4.2,3b1e610544a0 +-12.419,201,5.7,3b1e610544a0 +-12.308,201,5.2,3b1e610544a0 +-12.193,201,4.1,3b1e610544a0 +-12.079,201,4.6,b7b4f1a15dd3 +-11.965,201,4.5,b7b4f1a15dd3 +-11.855,201,6.4,3b1e610544a0 +-11.739,201,4.1,3b1e610544a0 +-11.625,201,5.8,3b1e610544a0 +-11.518,201,5.7,3b1e610544a0 +-11.404,201,6.8,b7b4f1a15dd3 +-11.288,201,4.9,b7b4f1a15dd3 +-11.174,201,6.6,b7b4f1a15dd3 +-11.059,201,5.6,3b1e610544a0 +-10.944,201,6.2,3b1e610544a0 +-10.827,201,6.2,b7b4f1a15dd3 +-10.714,201,6.4,b7b4f1a15dd3 +-10.606,201,5.3,b7b4f1a15dd3 +-10.497,201,4.8,b7b4f1a15dd3 +-10.383,201,4.8,3b1e610544a0 +-10.271,201,5.3,3b1e610544a0 +-10.163,201,4.0,b7b4f1a15dd3 +-10.057,201,5.0,b7b4f1a15dd3 +-9.946,201,4.8,b7b4f1a15dd3 +-9.831,201,5.6,3b1e610544a0 +-9.715,201,5.5,3b1e610544a0 +-9.604,201,5.0,3b1e610544a0 +-9.489,201,5.6,b7b4f1a15dd3 +-9.377,201,5.1,b7b4f1a15dd3 +-9.265,201,4.0,b7b4f1a15dd3 +-9.154,201,4.8,b7b4f1a15dd3 +-9.047,201,4.7,b7b4f1a15dd3 +-8.937,201,6.4,b7b4f1a15dd3 +-8.824,201,4.7,b7b4f1a15dd3 +-8.71,201,4.7,3b1e610544a0 +-8.598,201,6.0,3b1e610544a0 +-8.485,201,5.7,3b1e610544a0 +-8.378,201,5.3,3b1e610544a0 +-8.262,201,6.1,b7b4f1a15dd3 +-8.154,201,5.8,3b1e610544a0 +-8.048,201,6.4,3b1e610544a0 +-7.938,201,4.7,3b1e610544a0 +-7.825,201,6.6,3b1e610544a0 +-7.713,201,4.0,b7b4f1a15dd3 +-7.599,201,3.4,b7b4f1a15dd3 +-7.488,201,5.2,b7b4f1a15dd3 +-7.38,201,6.5,b7b4f1a15dd3 +-7.271,201,5.0,b7b4f1a15dd3 +-7.163,201,6.8,b7b4f1a15dd3 +-7.049,201,5.6,b7b4f1a15dd3 +-6.942,201,6.3,b7b4f1a15dd3 +-6.825,201,5.3,b7b4f1a15dd3 +-6.715,201,6.0,b7b4f1a15dd3 +-6.599,201,5.0,3b1e610544a0 +-6.488,201,4.8,3b1e610544a0 +-6.374,201,4.5,3b1e610544a0 +-6.259,201,4.6,b7b4f1a15dd3 +-6.144,201,4.5,b7b4f1a15dd3 +-6.033,201,5.4,3b1e610544a0 +-5.923,201,5.3,3b1e610544a0 +-5.811,201,4.6,3b1e610544a0 +-5.704,201,5.9,b7b4f1a15dd3 +-5.589,201,4.6,3b1e610544a0 +-5.479,201,5.6,3b1e610544a0 +-5.365,201,5.9,3b1e610544a0 +-5.257,201,5.2,3b1e610544a0 +-5.147,201,5.3,3b1e610544a0 +-5.035,201,5.3,3b1e610544a0 +-4.924,201,5.0,3b1e610544a0 +-4.811,201,5.5,b7b4f1a15dd3 +-4.696,201,5.4,b7b4f1a15dd3 +-4.59,201,6.6,b7b4f1a15dd3 +-4.481,201,5.2,b7b4f1a15dd3 +-4.372,201,6.3,b7b4f1a15dd3 +-4.264,201,5.4,3b1e610544a0 +-4.151,201,5.8,b7b4f1a15dd3 +-4.038,201,4.2,b7b4f1a15dd3 +-3.924,201,4.4,b7b4f1a15dd3 +-3.809,201,5.8,3b1e610544a0 +-3.693,201,5.5,b7b4f1a15dd3 +-3.579,201,4.8,b7b4f1a15dd3 +-3.471,201,5.2,b7b4f1a15dd3 +-3.356,201,7.8,b7b4f1a15dd3 +-3.246,201,7.2,b7b4f1a15dd3 +-3.138,201,5.8,b7b4f1a15dd3 +-3.028,201,5.8,b7b4f1a15dd3 +-2.917,201,4.9,b7b4f1a15dd3 +-2.802,201,5.4,3b1e610544a0 +-2.693,201,4.9,3b1e610544a0 +-2.581,201,4.0,3b1e610544a0 +-2.474,201,3.3,b7b4f1a15dd3 +-2.363,201,3.8,b7b4f1a15dd3 +-2.25,201,3.8,3b1e610544a0 +-2.144,201,4.8,b7b4f1a15dd3 +-2.037,201,5.2,b7b4f1a15dd3 +-1.922,201,4.5,3b1e610544a0 +-1.813,201,4.5,b7b4f1a15dd3 +-1.705,201,4.4,b7b4f1a15dd3 +-1.594,201,4.7,b7b4f1a15dd3 +-1.479,201,4.4,3b1e610544a0 +-1.365,201,4.5,3b1e610544a0 +-1.254,201,6.5,b7b4f1a15dd3 +-1.139,201,6.2,b7b4f1a15dd3 +-1.023,201,5.7,3b1e610544a0 +-0.911,201,5.9,3b1e610544a0 +-0.798,201,4.9,3b1e610544a0 +-0.692,201,5.4,3b1e610544a0 +-0.58,201,6.1,3b1e610544a0 +-0.471,201,5.9,3b1e610544a0 +-0.363,201,5.8,3b1e610544a0 +-0.249,201,5.2,3b1e610544a0 +-0.142,201,5.9,3b1e610544a0 +-0.027,201,4.2,3b1e610544a0 +0.078,201,4.6,3b1e610544a0 +0.185,201,4.3,b7b4f1a15dd3 +0.29,201,3.4,b7b4f1a15dd3 +0.394,201,3.3,3b1e610544a0 +0.502,201,3.3,3b1e610544a0 +0.616,201,4.0,3b1e610544a0 +0.721,201,3.1,3b1e610544a0 +0.828,201,5.7,3b1e610544a0 +0.943,201,4.2,3b1e610544a0 +1.055,201,4.7,3b1e610544a0 +1.161,201,4.9,3b1e610544a0 +1.269,201,3.5,3b1e610544a0 +1.376,201,3.4,b7b4f1a15dd3 +1.48,201,4.0,b7b4f1a15dd3 +1.595,201,5.0,b7b4f1a15dd3 +1.701,201,3.5,b7b4f1a15dd3 +1.812,201,3.6,b7b4f1a15dd3 +1.92,201,4.8,b7b4f1a15dd3 +2.034,201,3.8,b7b4f1a15dd3 +2.146,201,4.1,b7b4f1a15dd3 +2.258,201,7.3,3b1e610544a0 +2.365,201,4.4,b7b4f1a15dd3 +2.471,201,5.2,b7b4f1a15dd3 +2.576,201,3.6,3b1e610544a0 +2.689,201,4.2,b7b4f1a15dd3 +2.793,201,3.9,3b1e610544a0 +2.905,201,5.5,3b1e610544a0 +3.014,201,5.7,3b1e610544a0 +3.129,201,6.9,b7b4f1a15dd3 +3.237,201,4.9,b7b4f1a15dd3 +3.351,201,5.4,b7b4f1a15dd3 +3.466,201,4.8,b7b4f1a15dd3 +3.577,201,5.6,3b1e610544a0 +3.689,201,4.3,3b1e610544a0 +3.801,201,4.9,3b1e610544a0 +3.909,201,4.7,3b1e610544a0 +4.023,201,4.2,3b1e610544a0 +4.137,201,4.0,b7b4f1a15dd3 +4.245,201,5.4,3b1e610544a0 +4.36,201,4.6,b7b4f1a15dd3 +4.468,201,6.5,b7b4f1a15dd3 +4.581,201,5.2,b7b4f1a15dd3 +4.69,201,4.6,b7b4f1a15dd3 +4.805,201,6.3,b7b4f1a15dd3 +4.912,201,3.9,b7b4f1a15dd3 +5.023,201,3.7,b7b4f1a15dd3 +5.127,201,5.0,3b1e610544a0 +5.242,201,6.7,b7b4f1a15dd3 +5.359,201,5.6,96493b9901ab +5.474,201,5.4,96493b9901ab +5.589,201,5.5,3b1e610544a0 +5.697,201,6.3,96493b9901ab +5.804,201,5.2,96493b9901ab +5.911,201,6.6,96493b9901ab +6.019,201,7.3,96493b9901ab +6.132,201,4.8,96493b9901ab +6.247,201,5.5,3b1e610544a0 +6.356,201,5.6,3b1e610544a0 +6.469,201,5.5,96493b9901ab +6.582,201,4.5,96493b9901ab +6.696,201,5.4,96493b9901ab +6.809,201,5.1,3b1e610544a0 +6.92,201,5.9,3b1e610544a0 +7.029,201,6.6,3b1e610544a0 +7.145,201,5.7,3b1e610544a0 +7.261,201,5.4,96493b9901ab +7.372,201,3.9,96493b9901ab +7.478,201,6.1,96493b9901ab +7.587,201,4.7,3b1e610544a0 +7.692,201,3.7,3b1e610544a0 +7.803,201,5.5,3b1e610544a0 +7.918,201,3.6,96493b9901ab +8.031,201,4.1,96493b9901ab +8.14,201,4.2,96493b9901ab +8.245,201,4.8,3b1e610544a0 +8.352,201,4.2,3b1e610544a0 +8.459,201,4.7,3b1e610544a0 +8.566,201,3.7,96493b9901ab +8.671,201,3.7,96493b9901ab +8.777,201,5.0,96493b9901ab +8.892,201,4.2,96493b9901ab +9.0,201,3.8,96493b9901ab +9.11,201,5.2,96493b9901ab +9.216,201,4.1,96493b9901ab +9.329,201,4.5,96493b9901ab +9.442,201,6.1,96493b9901ab +9.553,201,4.9,96493b9901ab +9.668,201,5.3,3b1e610544a0 +9.778,201,5.4,3b1e610544a0 +9.894,201,5.5,3b1e610544a0 +10.004,201,4.6,3b1e610544a0 +10.113,201,3.6,3b1e610544a0 +10.222,201,3.0,3b1e610544a0 +10.328,201,3.1,3b1e610544a0 +10.44,201,2.8,3b1e610544a0 +10.546,201,4.1,96493b9901ab +10.66,201,3.5,3b1e610544a0 +10.769,201,4.1,3b1e610544a0 +10.884,201,5.3,3b1e610544a0 +10.994,201,4.7,96493b9901ab +11.104,201,4.4,96493b9901ab +11.208,201,4.3,96493b9901ab +11.313,201,4.5,3b1e610544a0 +11.42,201,4.6,3b1e610544a0 +11.533,201,6.9,3b1e610544a0 +11.644,201,4.5,3b1e610544a0 +11.749,201,3.7,3b1e610544a0 +11.853,201,3.1,dc1e7ee36921 +11.958,201,3.5,dc1e7ee36921 +12.072,201,5.1,96493b9901ab +12.182,201,3.5,96493b9901ab +12.294,201,6.4,96493b9901ab +12.401,201,4.0,96493b9901ab +12.508,201,5.4,96493b9901ab +12.62,201,3.8,96493b9901ab +12.732,201,3.3,96493b9901ab +12.84,201,3.1,96493b9901ab +12.943,201,4.1,dc1e7ee36921 +13.053,201,2.9,dc1e7ee36921 +13.162,201,3.6,dc1e7ee36921 +13.275,201,4.6,dc1e7ee36921 +13.383,201,4.4,dc1e7ee36921 +13.495,201,4.6,dc1e7ee36921 +13.604,201,3.7,dc1e7ee36921 +13.709,201,3.8,dc1e7ee36921 +13.815,201,4.6,96493b9901ab +13.923,201,3.4,96493b9901ab +14.028,201,5.2,96493b9901ab +14.135,201,5.4,dc1e7ee36921 +14.247,201,4.0,96493b9901ab +14.356,201,3.8,dc1e7ee36921 +14.463,201,4.5,dc1e7ee36921 +14.578,201,5.0,96493b9901ab +14.691,201,5.4,96493b9901ab +14.802,201,4.2,96493b9901ab +14.915,201,3.0,96493b9901ab +15.02,201,4.2,96493b9901ab +15.128,201,5.2,96493b9901ab +15.238,201,4.6,96493b9901ab +15.343,201,4.8,dc1e7ee36921 +15.452,201,5.2,dc1e7ee36921 +15.562,201,4.8,dc1e7ee36921 +15.676,201,5.1,96493b9901ab +15.783,201,4.9,dc1e7ee36921 +15.889,201,5.3,dc1e7ee36921 +16.003,201,4.5,dc1e7ee36921 +16.109,201,5.0,dc1e7ee36921 +16.223,201,4.5,dc1e7ee36921 +16.338,201,6.6,96493b9901ab +16.447,201,5.2,96493b9901ab +16.561,201,5.3,96493b9901ab +16.669,201,6.4,96493b9901ab +16.78,201,4.7,96493b9901ab +16.889,201,5.3,96493b9901ab +17.003,201,5.9,96493b9901ab +17.116,201,5.6,dc1e7ee36921 +17.227,201,5.8,dc1e7ee36921 +17.343,201,5.3,dc1e7ee36921 +17.448,201,5.1,dc1e7ee36921 +17.557,201,6.4,dc1e7ee36921 +17.665,201,5.7,96493b9901ab +17.781,201,5.1,dc1e7ee36921 +17.895,201,3.5,dc1e7ee36921 +18.006,201,4.9,96493b9901ab +18.12,201,5.0,96493b9901ab +18.228,201,5.8,96493b9901ab +18.34,201,5.6,96493b9901ab +18.456,201,5.6,96493b9901ab +18.571,201,4.9,dc1e7ee36921 +18.678,201,5.3,dc1e7ee36921 +18.788,201,5.6,dc1e7ee36921 +18.9,201,5.4,dc1e7ee36921 +19.007,201,4.2,dc1e7ee36921 +19.114,201,4.2,96493b9901ab +19.223,201,5.2,96493b9901ab +19.336,201,5.5,dc1e7ee36921 +19.445,201,4.9,96493b9901ab +19.56,201,5.5,dc1e7ee36921 +19.669,201,6.1,dc1e7ee36921 +19.78,201,6.8,dc1e7ee36921 +19.895,201,5.8,dc1e7ee36921 +20.006,201,7.2,96493b9901ab +20.123,201,4.8,96493b9901ab +20.237,201,4.7,dc1e7ee36921 +20.345,201,4.6,96493b9901ab +20.459,201,6.4,dc1e7ee36921 +20.574,201,5.3,dc1e7ee36921 +20.687,201,5.5,96493b9901ab +20.795,201,6.0,dc1e7ee36921 +20.907,201,5.4,96493b9901ab +21.017,201,4.5,96493b9901ab +21.126,201,7.0,96493b9901ab +21.236,201,5.3,96493b9901ab +21.345,201,6.2,dc1e7ee36921 +21.46,201,7.3,96493b9901ab +21.577,201,4.9,dc1e7ee36921 +21.686,201,5.4,dc1e7ee36921 +21.801,201,5.1,dc1e7ee36921 +21.916,201,6.3,96493b9901ab +22.028,201,6.4,96493b9901ab +22.142,201,5.9,96493b9901ab +22.253,201,4.1,96493b9901ab +22.361,201,5.0,96493b9901ab +22.47,201,5.9,96493b9901ab +22.586,201,5.0,dc1e7ee36921 +22.699,201,7.0,dc1e7ee36921 +22.812,201,4.9,dc1e7ee36921 +22.924,201,5.9,dc1e7ee36921 +23.04,201,3.8,96493b9901ab +23.154,201,4.4,96493b9901ab +23.262,201,6.0,96493b9901ab +23.374,201,5.1,96493b9901ab +23.489,201,5.7,dc1e7ee36921 +23.596,201,6.3,96493b9901ab +23.704,201,6.2,96493b9901ab +23.82,201,4.1,dc1e7ee36921 +23.934,201,5.6,dc1e7ee36921 +24.041,201,3.2,dc1e7ee36921 +24.153,201,4.6,dc1e7ee36921 +24.267,201,4.0,96493b9901ab +24.381,201,3.7,96493b9901ab +24.495,201,4.6,96493b9901ab +24.609,201,5.6,dc1e7ee36921 +24.725,201,5.8,dc1e7ee36921 +24.832,201,5.8,dc1e7ee36921 +24.944,201,4.7,dc1e7ee36921 +25.053,201,4.9,dc1e7ee36921 +25.161,201,6.1,dc1e7ee36921 +25.27,201,5.6,dc1e7ee36921 +25.383,201,5.7,dc1e7ee36921 +25.495,201,6.1,dc1e7ee36921 +25.607,201,5.7,96493b9901ab +25.719,201,5.0,96493b9901ab +25.825,201,6.4,96493b9901ab +25.941,201,5.0,96493b9901ab +26.052,201,4.3,96493b9901ab +26.161,201,5.9,96493b9901ab +26.269,201,5.4,96493b9901ab +26.377,201,5.6,96493b9901ab +26.487,201,3.3,96493b9901ab +26.595,201,4.7,dc1e7ee36921 +26.709,201,5.1,96493b9901ab +26.824,201,5.2,96493b9901ab +26.939,201,5.4,dc1e7ee36921 +27.055,201,5.9,96493b9901ab +27.168,201,6.2,96493b9901ab +27.28,201,4.6,96493b9901ab +27.394,201,5.0,96493b9901ab +27.505,201,5.8,96493b9901ab +27.619,201,5.3,dc1e7ee36921 +27.727,201,5.1,dc1e7ee36921 +27.837,201,5.1,dc1e7ee36921 +27.945,201,4.2,96493b9901ab +28.049,201,6.2,96493b9901ab +28.159,201,4.0,96493b9901ab +28.269,201,3.0,96493b9901ab +28.381,201,3.3,96493b9901ab +28.486,201,4.9,dc1e7ee36921 +28.594,201,4.1,dc1e7ee36921 +28.7,201,4.8,dc1e7ee36921 +28.812,201,4.5,dc1e7ee36921 +28.919,201,4.6,dc1e7ee36921 +29.027,201,7.0,dc1e7ee36921 +29.14,201,5.8,dc1e7ee36921 +29.256,201,4.2,dc1e7ee36921 +29.367,201,4.2,96493b9901ab +29.481,201,5.0,96493b9901ab +29.594,201,4.9,96493b9901ab +29.706,201,7.5,dc1e7ee36921 +29.813,201,4.9,96493b9901ab +29.926,201,5.7,dc1e7ee36921 +30.042,201,5.9,dc1e7ee36921 +30.158,201,7.1,96493b9901ab +30.271,201,5.1,96493b9901ab +30.386,201,5.6,dc1e7ee36921 +30.493,201,8.1,96493b9901ab +30.607,201,5.1,dc1e7ee36921 +30.718,201,5.9,dc1e7ee36921 +30.831,201,6.7,dc1e7ee36921 +30.944,201,5.0,dc1e7ee36921 +31.051,201,6.2,dc1e7ee36921 +31.163,201,6.3,dc1e7ee36921 +31.276,201,6.4,96493b9901ab +31.387,201,5.6,dc1e7ee36921 +31.495,201,7.2,96493b9901ab +31.61,201,4.4,96493b9901ab +31.723,201,5.1,96493b9901ab +31.837,201,4.5,dc1e7ee36921 +31.947,201,5.1,dc1e7ee36921 +32.061,201,6.6,dc1e7ee36921 +32.169,201,5.1,dc1e7ee36921 +32.281,201,5.9,dc1e7ee36921 +32.394,201,5.5,dc1e7ee36921 +32.509,201,4.8,96493b9901ab +32.624,201,4.9,96493b9901ab +32.737,201,4.9,dc1e7ee36921 +32.846,201,6.2,96493b9901ab +32.953,201,4.5,96493b9901ab +33.067,201,5.2,dc1e7ee36921 +33.183,201,6.3,dc1e7ee36921 +33.298,201,12.8,96493b9901ab +33.419,201,3.9,dc1e7ee36921 +33.526,201,5.4,dc1e7ee36921 +33.64,201,3.7,dc1e7ee36921 +33.753,201,6.1,dc1e7ee36921 +33.861,201,5.4,dc1e7ee36921 +33.968,201,4.7,dc1e7ee36921 +34.083,201,3.9,dc1e7ee36921 +34.197,201,5.1,96493b9901ab +34.312,201,6.0,96493b9901ab +34.421,201,4.7,96493b9901ab +34.534,201,5.8,96493b9901ab +34.65,201,4.5,dc1e7ee36921 +34.756,201,6.1,dc1e7ee36921 +34.863,201,5.0,96493b9901ab +34.974,201,5.0,96493b9901ab +35.089,201,5.4,dc1e7ee36921 +35.2,201,5.3,dc1e7ee36921 +35.314,201,6.1,dc1e7ee36921 +35.428,201,5.3,96493b9901ab +35.533,201,4.8,dc1e7ee36921 +35.648,201,4.8,96493b9901ab +35.756,201,5.1,96493b9901ab +35.864,201,5.4,dc1e7ee36921 +35.979,201,5.7,96493b9901ab +36.092,201,6.6,96493b9901ab +36.206,201,6.7,dc1e7ee36921 +36.316,201,4.9,96493b9901ab +36.431,201,4.8,dc1e7ee36921 +36.536,201,5.2,dc1e7ee36921 +36.644,201,4.9,96493b9901ab +36.759,201,4.8,dc1e7ee36921 +36.866,201,5.7,dc1e7ee36921 +36.979,201,5.6,dc1e7ee36921 +37.092,201,5.1,dc1e7ee36921 +37.208,201,4.7,96493b9901ab +37.321,201,4.9,96493b9901ab +37.431,201,4.9,96493b9901ab +37.544,201,5.0,96493b9901ab +37.649,201,5.6,96493b9901ab +37.755,201,5.1,96493b9901ab +37.869,201,5.2,96493b9901ab +37.984,201,4.8,96493b9901ab +38.098,201,5.0,dc1e7ee36921 +38.213,201,5.5,dc1e7ee36921 +38.329,201,6.0,96493b9901ab +38.443,201,4.0,96493b9901ab +38.551,201,4.9,96493b9901ab +38.66,201,5.2,96493b9901ab +38.769,201,4.4,96493b9901ab +38.884,201,6.3,dc1e7ee36921 +38.994,201,5.6,96493b9901ab +39.109,201,4.4,dc1e7ee36921 +39.219,201,5.3,dc1e7ee36921 +39.333,201,6.0,dc1e7ee36921 +39.442,201,5.2,dc1e7ee36921 +39.549,201,3.6,dc1e7ee36921 +39.654,201,4.0,96493b9901ab +39.767,201,5.1,dc1e7ee36921 +39.882,201,6.4,dc1e7ee36921 +39.994,201,5.4,dc1e7ee36921 +40.109,201,5.3,96493b9901ab +40.221,201,5.5,96493b9901ab +40.336,201,4.9,dc1e7ee36921 +40.45,201,5.5,dc1e7ee36921 +40.561,201,5.2,dc1e7ee36921 +40.669,201,7.3,dc1e7ee36921 +40.778,201,5.1,dc1e7ee36921 +40.891,201,6.8,dc1e7ee36921 +41.005,201,5.5,96493b9901ab +41.119,201,5.5,96493b9901ab +41.229,201,5.0,96493b9901ab +41.342,201,5.1,96493b9901ab +41.457,201,6.5,dc1e7ee36921 +41.574,201,5.8,dc1e7ee36921 +41.685,201,6.3,96493b9901ab +41.796,201,7.0,96493b9901ab +41.911,201,5.3,96493b9901ab +42.018,201,4.3,96493b9901ab +42.133,201,5.0,96493b9901ab +42.248,201,5.8,dc1e7ee36921 +42.357,201,5.5,dc1e7ee36921 +42.469,201,5.0,dc1e7ee36921 +42.577,201,5.9,dc1e7ee36921 +42.684,201,4.9,96493b9901ab +42.795,201,4.5,dc1e7ee36921 +42.902,201,5.1,96493b9901ab +43.01,201,5.0,96493b9901ab +43.124,201,5.1,96493b9901ab +43.235,201,4.8,96493b9901ab +43.344,201,5.1,96493b9901ab +43.45,201,4.6,96493b9901ab +43.555,201,4.2,dc1e7ee36921 +43.669,201,3.0,96493b9901ab +43.781,201,3.9,96493b9901ab +43.891,201,5.6,96493b9901ab +44.0,201,4.9,96493b9901ab +44.111,201,5.2,96493b9901ab +44.219,201,5.3,96493b9901ab +44.325,201,4.9,96493b9901ab +44.431,201,6.1,dc1e7ee36921 +44.544,201,6.5,96493b9901ab +44.653,201,4.2,dc1e7ee36921 +44.763,201,4.7,dc1e7ee36921 +44.872,201,4.0,dc1e7ee36921 +44.983,201,5.9,dc1e7ee36921 +45.094,201,6.2,dc1e7ee36921 +45.205,201,5.2,dc1e7ee36921 +45.316,201,5.5,96493b9901ab +45.424,201,5.2,dc1e7ee36921 +45.539,201,5.1,96493b9901ab +45.651,201,5.8,96493b9901ab +45.767,201,5.1,dc1e7ee36921 +45.882,201,6.4,dc1e7ee36921 +45.999,201,5.3,96493b9901ab +46.113,201,6.1,96493b9901ab +46.227,201,5.8,dc1e7ee36921 +46.335,201,5.8,dc1e7ee36921 +46.448,201,5.7,dc1e7ee36921 +46.555,201,6.2,dc1e7ee36921 +46.667,201,5.7,dc1e7ee36921 +46.774,201,5.0,96493b9901ab +46.888,201,5.9,dc1e7ee36921 +47.001,201,6.5,dc1e7ee36921 +47.112,201,7.5,dc1e7ee36921 +47.23,201,5.3,96493b9901ab +47.344,201,5.7,96493b9901ab +47.453,201,4.9,96493b9901ab +47.567,201,5.4,dc1e7ee36921 +47.679,201,5.2,dc1e7ee36921 +47.788,201,5.4,dc1e7ee36921 +47.901,201,6.4,dc1e7ee36921 +48.015,201,5.3,96493b9901ab +48.129,201,4.9,96493b9901ab +48.238,201,4.8,96493b9901ab +48.351,201,5.2,96493b9901ab +48.46,201,5.0,96493b9901ab +48.575,201,5.8,dc1e7ee36921 +48.688,201,4.0,dc1e7ee36921 +48.792,201,3.7,96493b9901ab +48.906,201,3.8,dc1e7ee36921 +49.011,201,5.7,96493b9901ab +49.124,201,5.3,96493b9901ab +49.235,201,5.1,96493b9901ab +49.346,201,5.2,dc1e7ee36921 +49.459,201,4.5,dc1e7ee36921 +49.571,201,4.7,dc1e7ee36921 +49.677,201,5.3,dc1e7ee36921 +49.787,201,4.1,dc1e7ee36921 +49.897,201,3.1,dc1e7ee36921 +50.004,201,4.0,96493b9901ab +50.118,201,5.3,dc1e7ee36921 +50.234,201,5.0,dc1e7ee36921 +50.346,201,7.0,96493b9901ab +50.46,201,4.9,96493b9901ab +50.568,201,5.2,96493b9901ab +50.675,201,4.9,dc1e7ee36921 +50.79,201,4.6,96493b9901ab +50.904,201,4.6,96493b9901ab +51.01,201,4.7,96493b9901ab +51.119,201,6.2,96493b9901ab +51.236,201,5.6,96493b9901ab +51.35,201,4.8,dc1e7ee36921 +51.465,201,5.9,96493b9901ab +51.581,201,5.8,96493b9901ab +51.693,201,5.5,96493b9901ab +51.802,201,5.5,96493b9901ab +51.918,201,6.5,dc1e7ee36921 +52.035,201,5.7,96493b9901ab +52.15,201,5.1,96493b9901ab +52.256,201,7.2,96493b9901ab +52.365,201,4.5,96493b9901ab +52.47,201,5.4,dc1e7ee36921 +52.577,201,4.9,dc1e7ee36921 +52.692,201,5.4,dc1e7ee36921 +52.806,201,5.5,96493b9901ab +52.921,201,5.0,96493b9901ab +53.027,201,5.0,96493b9901ab +53.142,201,6.5,96493b9901ab +53.251,201,6.8,96493b9901ab +53.366,201,5.3,dc1e7ee36921 +53.474,201,5.7,dc1e7ee36921 +53.581,201,5.5,96493b9901ab +53.69,201,5.0,96493b9901ab +53.796,201,5.6,96493b9901ab +53.905,201,3.3,dc1e7ee36921 +54.009,201,5.1,dc1e7ee36921 +54.121,201,4.3,dc1e7ee36921 +54.236,201,5.0,96493b9901ab +54.347,201,5.0,96493b9901ab +54.462,201,4.8,96493b9901ab +54.577,201,4.6,dc1e7ee36921 +54.69,201,4.8,dc1e7ee36921 +54.798,201,5.3,dc1e7ee36921 +54.911,201,6.4,dc1e7ee36921 +55.026,201,3.4,96493b9901ab +55.13,201,3.6,dc1e7ee36921 +55.24,201,4.7,dc1e7ee36921 +55.353,201,5.9,dc1e7ee36921 +55.468,201,5.3,96493b9901ab +55.584,201,4.9,96493b9901ab +55.69,201,6.5,96493b9901ab +55.802,201,5.7,96493b9901ab +55.912,201,5.0,96493b9901ab +56.019,201,5.9,96493b9901ab +56.125,201,5.8,96493b9901ab +56.235,201,5.5,96493b9901ab +56.35,201,6.0,96493b9901ab +56.466,201,5.5,dc1e7ee36921 +56.576,201,6.4,dc1e7ee36921 +56.688,201,6.0,dc1e7ee36921 +56.804,201,5.9,dc1e7ee36921 +56.92,201,4.9,96493b9901ab +57.025,201,5.1,96493b9901ab +57.135,201,5.9,96493b9901ab +57.243,201,6.8,dc1e7ee36921 +57.358,201,6.2,96493b9901ab +57.474,201,4.7,96493b9901ab +57.583,201,5.0,96493b9901ab +57.696,201,6.2,dc1e7ee36921 +57.81,201,4.2,dc1e7ee36921 +57.922,201,4.8,dc1e7ee36921 +58.032,201,4.8,dc1e7ee36921 +58.145,201,5.1,96493b9901ab +58.252,201,5.2,dc1e7ee36921 +58.359,201,10.0,dc1e7ee36921 +58.477,201,4.9,96493b9901ab +58.592,201,4.5,96493b9901ab +58.704,201,4.8,96493b9901ab +58.81,201,4.6,96493b9901ab +58.919,201,4.9,96493b9901ab +59.031,201,3.8,96493b9901ab +59.135,201,3.5,96493b9901ab +59.239,201,4.0,dc1e7ee36921 +59.351,201,6.5,dc1e7ee36921 +59.467,201,6.1,96493b9901ab +59.583,201,5.2,96493b9901ab +59.698,201,4.9,dc1e7ee36921 +59.808,201,5.3,dc1e7ee36921 +59.913,201,5.6,96493b9901ab +60.021,201,5.2,96493b9901ab +60.133,201,3.7,96493b9901ab +60.243,201,5.4,96493b9901ab +60.359,201,5.0,dc1e7ee36921 +60.468,201,6.2,dc1e7ee36921 +60.576,201,5.3,dc1e7ee36921 +60.682,201,6.2,96493b9901ab +60.794,201,5.2,96493b9901ab +60.903,201,5.5,96493b9901ab +61.018,201,4.2,dc1e7ee36921 +61.125,201,6.0,dc1e7ee36921 +61.233,201,6.9,96493b9901ab +61.343,201,5.3,96493b9901ab +61.453,201,6.9,96493b9901ab +61.569,201,6.4,dc1e7ee36921 +61.677,201,4.0,dc1e7ee36921 +61.791,201,4.8,dc1e7ee36921 +61.906,201,5.2,96493b9901ab +62.019,201,5.7,96493b9901ab +62.126,201,5.3,96493b9901ab +62.242,201,5.0,96493b9901ab +62.353,201,6.4,96493b9901ab +62.46,201,5.3,96493b9901ab +62.573,201,6.3,96493b9901ab +62.689,201,5.0,dc1e7ee36921 +62.802,201,6.2,dc1e7ee36921 +62.912,201,5.7,dc1e7ee36921 +63.026,201,5.0,96493b9901ab +63.136,201,6.5,96493b9901ab +63.243,201,5.1,dc1e7ee36921 +63.357,201,4.9,96493b9901ab +63.47,201,5.1,96493b9901ab +63.583,201,5.7,96493b9901ab +63.69,201,5.0,96493b9901ab +63.804,201,5.6,dc1e7ee36921 +63.912,201,6.4,96493b9901ab +64.026,201,5.3,dc1e7ee36921 +64.141,201,4.8,dc1e7ee36921 +64.256,201,3.7,96493b9901ab +64.362,201,5.0,dc1e7ee36921 +64.468,201,5.2,dc1e7ee36921 +64.583,201,5.6,dc1e7ee36921 +64.698,201,6.7,96493b9901ab +64.812,201,6.6,96493b9901ab +64.926,201,6.4,dc1e7ee36921 +65.042,201,4.3,dc1e7ee36921 +65.156,201,5.0,96493b9901ab +65.261,201,4.2,dc1e7ee36921 +65.369,201,3.7,dc1e7ee36921 +65.476,201,4.7,dc1e7ee36921 +65.582,201,6.5,96493b9901ab +65.694,201,5.6,96493b9901ab +65.81,201,5.7,dc1e7ee36921 +65.921,201,6.1,dc1e7ee36921 +66.033,201,6.2,dc1e7ee36921 +66.149,201,5.1,96493b9901ab +66.259,201,5.7,96493b9901ab +66.367,201,6.5,96493b9901ab +66.484,201,5.1,dc1e7ee36921 +66.593,201,5.1,96493b9901ab +66.709,201,5.2,dc1e7ee36921 +66.824,201,3.1,dc1e7ee36921 +66.937,201,3.5,96493b9901ab +67.042,201,3.8,dc1e7ee36921 +67.147,201,3.9,dc1e7ee36921 +67.261,201,3.6,dc1e7ee36921 +67.367,201,4.8,dc1e7ee36921 +67.476,201,5.4,dc1e7ee36921 +67.592,201,3.2,dc1e7ee36921 +67.702,201,3.8,dc1e7ee36921 +67.812,201,4.7,dc1e7ee36921 +67.926,201,3.5,96493b9901ab +68.04,201,5.8,96493b9901ab +68.149,201,4.9,96493b9901ab +68.259,201,6.4,96493b9901ab +68.371,201,6.3,96493b9901ab +68.487,201,5.0,dc1e7ee36921 +68.602,201,5.1,dc1e7ee36921 +68.709,201,6.8,dc1e7ee36921 +68.826,201,5.4,96493b9901ab +68.935,201,4.7,96493b9901ab +69.041,201,4.3,dc1e7ee36921 +69.154,201,3.5,dc1e7ee36921 +69.258,201,2.8,dc1e7ee36921 +69.361,201,2.9,96493b9901ab +69.468,201,3.2,96493b9901ab +69.579,201,3.7,96493b9901ab +69.684,201,3.6,dc1e7ee36921 +69.793,201,3.6,dc1e7ee36921 +69.899,201,5.0,dc1e7ee36921 +70.01,201,7.8,dc1e7ee36921 +70.122,201,4.2,dc1e7ee36921 +70.236,201,4.8,96493b9901ab +70.343,201,6.6,dc1e7ee36921 +70.459,201,3.3,96493b9901ab +70.568,201,3.0,96493b9901ab +70.679,201,3.7,96493b9901ab +70.791,201,3.2,96493b9901ab +70.903,201,3.6,96493b9901ab +71.013,201,6.0,96493b9901ab +71.122,201,6.1,96493b9901ab +71.23,201,4.2,96493b9901ab +71.343,201,5.1,96493b9901ab +71.45,201,4.6,96493b9901ab +71.562,201,5.1,96493b9901ab +71.67,201,4.9,96493b9901ab +71.784,201,13.2,dc1e7ee36921 +71.904,201,6.0,96493b9901ab +72.018,201,4.5,96493b9901ab +72.132,201,5.2,96493b9901ab +72.248,201,5.3,dc1e7ee36921 +72.355,201,5.9,dc1e7ee36921 +72.468,201,6.0,dc1e7ee36921 +72.583,201,6.1,dc1e7ee36921 +72.699,201,5.4,96493b9901ab +72.807,201,5.6,96493b9901ab +72.922,201,5.1,96493b9901ab +73.032,201,6.2,96493b9901ab +73.143,201,6.2,96493b9901ab +73.251,201,4.3,96493b9901ab +73.365,201,5.4,dc1e7ee36921 +73.481,201,7.1,dc1e7ee36921 +73.591,201,6.8,dc1e7ee36921 +73.707,201,3.8,96493b9901ab +73.821,201,3.5,96493b9901ab +73.934,201,5.7,dc1e7ee36921 +74.043,201,4.6,96493b9901ab +74.153,201,5.1,96493b9901ab +74.268,201,5.3,dc1e7ee36921 +74.376,201,5.4,dc1e7ee36921 +74.491,201,3.5,dc1e7ee36921 +74.605,201,3.5,96493b9901ab +74.718,201,5.4,96493b9901ab +74.828,201,5.5,96493b9901ab +74.941,201,6.0,96493b9901ab +75.051,201,5.6,dc1e7ee36921 +75.161,201,8.1,96493b9901ab +75.276,201,5.5,dc1e7ee36921 +75.389,201,7.8,96493b9901ab +75.503,201,5.2,dc1e7ee36921 diff --git a/evidence/rolling-from-7fc911c/result.json b/evidence/rolling-from-7fc911c/result.json new file mode 100644 index 00000000..39ea2783 --- /dev/null +++ b/evidence/rolling-from-7fc911c/result.json @@ -0,0 +1,66 @@ +{"date": "2026-11-05T19:22:59.139702+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", + "label": "action", + "action": [ + "scripts/rolling-update.sh" + ], + "action_exit": 0, + "action_seconds": 13.836, + "action_finished_before_load_end": true, + "lb_up_before": 2, + "lb_up_min": 2, + "lb_samples_missing": 0, + "detection_seconds": null, + "rejoin_seconds": null, + "probe": { + "attempts": 913, + "failed_attempts": 0, + "first_failure_s": null, + "first_success_s": 0.082, + "recovery_s": 0.0, + "instances_before": { + "b7b4f1a15dd3": 106, + "3b1e610544a0": 127 + }, + "instances_during_outage": {}, + "instances_after": { + "96493b9901ab": 291, + "dc1e7ee36921": 263 + } + }, + "history": { + "confirmed": 913, + "stored": 913, + "exact_match": true, + "unique": true + }, + "lb_counters": { + "wretr": 0, + "wredis": 0, + "econ": 0, + "eresp": 0, + "hrsp_5xx": 0 + }, + "k6": { + "requests": 9001, + "rps": 100.01, + "ok_rps": 100.01, + "failed_ratio": 0, + "dropped_iterations": 0, + "p50_ms": 2.99, + "p95_ms": 3.85, + "p99_ms": 4.47, + "exit": 0, + "rate": 100 + }, + "checks": { + "action_ok": true, + "load_errors_ok": true, + "load_delivered": true, + "lb_observed": true, + "write_recovery_ok": true, + "history_preserved": true, + "capacity_kept": true, + "all_replaced": true + }, + "pass": true +} diff --git a/evidence/rolling/action.txt b/evidence/rolling/action.txt index f66d569e..570c7274 100644 --- a/evidence/rolling/action.txt +++ b/evidence/rolling/action.txt @@ -1,35 +1,35 @@ -02:52:53 migrate once before the first replacement -02:52:53 database bytes,messages: 20534963,24025 -{"reason":"up to date","event":"migration.completed","level":"info","ts":"2026-10-30T02:52:54.219740Z"} - Container mtusi2026lab3-api-3 Running - Container mtusi2026lab3-api-4 Running - Container mtusi2026lab3-api-5 Creating - Container mtusi2026lab3-api-5 Created - Container mtusi2026lab3-api-5 Starting - Container mtusi2026lab3-api-5 Started - Container mtusi2026lab3-api-5 Waiting - Container mtusi2026lab3-api-4 Waiting - Container mtusi2026lab3-api-3 Waiting - Container mtusi2026lab3-api-5 Healthy - Container mtusi2026lab3-api-4 Healthy - Container mtusi2026lab3-api-3 Healthy -mtusi2026lab3-api-3 cpu=18.37% mem=160.3MiB / 512MiB -mtusi2026lab3-api-4 cpu=5.61% mem=160.3MiB / 512MiB -mtusi2026lab3-lb-1 cpu=2.95% mem=21.96MiB / 128MiB -mtusi2026lab3-postgres-1 cpu=5.48% mem=68.1MiB / 1GiB -mtusi2026lab3-valkey-1 cpu=0.42% mem=13.11MiB / 128MiB -02:52:56 replace 6c2a02267340 image=mtusi-messenger:mtusi2026lab3@c69a721bc974 -> c2f5d82dee7c image=mtusi-messenger:mtusi2026lab3@021faac1e94a - Container mtusi2026lab3-api-4 Running - Container mtusi2026lab3-api-5 Running - Container mtusi2026lab3-api-6 Creating - Container mtusi2026lab3-api-6 Created - Container mtusi2026lab3-api-6 Starting - Container mtusi2026lab3-api-6 Started - Container mtusi2026lab3-api-6 Waiting - Container mtusi2026lab3-api-5 Waiting - Container mtusi2026lab3-api-4 Waiting - Container mtusi2026lab3-api-6 Healthy - Container mtusi2026lab3-api-5 Healthy - Container mtusi2026lab3-api-4 Healthy -02:53:03 replace 158dfc1bd95d image=mtusi-messenger:mtusi2026lab3@c69a721bc974 -> 65d10d346f4e image=mtusi-messenger:mtusi2026lab3@021faac1e94a -02:53:05 done: 2 replicas replaced +date: 2026-11-05T19:21:07.986361+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +19:21:34 migrate once before the first replacement +19:21:34 database bytes,messages: 15718067,14730 + Container mtusi2026lab3-grade4-20261105-api-1 Running + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-3 Creating + Container mtusi2026lab3-grade4-20261105-api-3 Created + Container mtusi2026lab3-grade4-20261105-api-3 Starting + Container mtusi2026lab3-grade4-20261105-api-3 Started + Container mtusi2026lab3-grade4-20261105-api-3 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-1 Waiting + Container mtusi2026lab3-grade4-20261105-api-3 Healthy + Container mtusi2026lab3-grade4-20261105-api-2 Healthy + Container mtusi2026lab3-grade4-20261105-api-1 Healthy +mtusi2026lab3-grade4-20261105-api-1 cpu=10.09% mem=220.2MiB / 512MiB +mtusi2026lab3-grade4-20261105-api-2 cpu=11.47% mem=210.3MiB / 512MiB +mtusi2026lab3-grade4-20261105-lb-1 cpu=2.70% mem=30.95MiB / 128MiB +mtusi2026lab3-grade4-20261105-postgres-1 cpu=4.20% mem=70.17MiB / 1GiB +mtusi2026lab3-grade4-20261105-valkey-1 cpu=0.37% mem=13.22MiB / 128MiB +19:21:38 replace 7cfd91ddf780 image=mtusi-messenger:mtusi2026lab3-grade4-20261105@fdc0128530c7 -> a352bb0a17ff image=mtusi-messenger:mtusi2026lab3-grade4-20261105@17d0f0e40288 + Container mtusi2026lab3-grade4-20261105-api-2 Running + Container mtusi2026lab3-grade4-20261105-api-3 Running + Container mtusi2026lab3-grade4-20261105-api-4 Creating + Container mtusi2026lab3-grade4-20261105-api-4 Created + Container mtusi2026lab3-grade4-20261105-api-4 Starting + Container mtusi2026lab3-grade4-20261105-api-4 Started + Container mtusi2026lab3-grade4-20261105-api-4 Waiting + Container mtusi2026lab3-grade4-20261105-api-3 Waiting + Container mtusi2026lab3-grade4-20261105-api-2 Waiting + Container mtusi2026lab3-grade4-20261105-api-4 Healthy + Container mtusi2026lab3-grade4-20261105-api-3 Healthy + Container mtusi2026lab3-grade4-20261105-api-2 Healthy +19:21:42 replace 5bbf259ff136 image=mtusi-messenger:mtusi2026lab3-grade4-20261105@fdc0128530c7 -> f6a858d7a20b image=mtusi-messenger:mtusi2026lab3-grade4-20261105@17d0f0e40288 +19:21:45 done: 2 replicas replaced diff --git a/evidence/rolling/k6/k6.txt b/evidence/rolling/k6/k6.txt index 5da226ff..9da5b6bf 100644 --- a/evidence/rolling/k6/k6.txt +++ b/evidence/rolling/k6/k6.txt @@ -1,2 +1,3 @@ -time="2026-10-30T02:52:28Z" level=info msg=LOAD-START source=console +date: 2026-11-05T19:21:07.986361+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 +time="2026-11-05T19:21:09Z" level=info msg=LOAD-START source=console measure: 9001 requests, failed ratio 0 diff --git a/evidence/rolling/k6/summary.json b/evidence/rolling/k6/summary.json index 829cdf05..bf7124cb 100644 --- a/evidence/rolling/k6/summary.json +++ b/evidence/rolling/k6/summary.json @@ -1,309 +1,309 @@ -{ - "root_group": { - "checks": [ - { - "path": "::message 201", - "id": "6445aff337e1ade0bc10402fbac79882", - "passes": 8049, - "fails": 0, - "name": "message 201" - }, - { - "fails": 0, - "name": "history 200", - "path": "::history 200", - "id": "fe27b3b8edd292c7f2170a6357b087d0", - "passes": 1953 - } - ], - "name": "", - "path": "", - "id": "d41d8cd98f00b204e9800998ecf8427e", - "groups": [] - }, - "options": { - "summaryTrendStats": [ - "avg", - "min", - "med", - "p(90)", - "p(95)", - "p(99)", - "max" - ], - "summaryTimeUnit": "", - "noColor": false - }, - "state": { - "isStdOutTTY": false, - "isStdErrTTY": false, - "testRunDurationMs": 100562.669839 - }, - "metrics": { - "iteration_duration": { - "values": { - "p(99)": 6.5293851699999985, - "max": 59.366542, - "avg": 1.6908107441511713, - "min": 0.937167, - "med": 1.4179995, - "p(90)": 2.0306586, - "p(95)": 2.3185982499999986 - }, - "type": "trend", - "contains": "time" +{"date": "2026-11-05T19:21:07.986361+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "path": "::message 201", + "id": "6445aff337e1ade0bc10402fbac79882", + "passes": 7942, + "fails": 0, + "name": "message 201" + }, + { + "name": "history 200", + "path": "::history 200", + "id": "fe27b3b8edd292c7f2170a6357b087d0", + "passes": 2060, + "fails": 0 + } + ] }, - "http_reqs{scenario:measure}": { - "contains": "default", - "values": { - "rate": 89.50637462599717, - "count": 9001 - }, - "type": "counter" + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "p(90)", + "p(95)", + "p(99)", + "max" + ], + "summaryTimeUnit": "", + "noColor": false }, - "iterations{scenario:measure}": { - "type": "counter", - "contains": "default", - "values": { - "count": 9001, - "rate": 89.50637462599717 - } + "state": { + "isStdOutTTY": false, + "isStdErrTTY": false, + "testRunDurationMs": 100594.316839 }, - "http_req_connecting": { - "type": "trend", - "contains": "time", - "values": { - "avg": 0.0003895038798249105, - "min": 0, - "med": 0, - "p(90)": 0, - "p(95)": 0, - "p(99)": 0, - "max": 0.258041 - } - }, - "http_req_waiting": { - "type": "trend", - "contains": "time", - "values": { - "min": 0.846458, - "med": 1.3211455, - "p(90)": 1.8879506, - "p(95)": 2.198289799999999, - "p(99)": 8.954823409999985, - "max": 59.019459, - "avg": 1.626115160067644 - } - }, - "http_req_receiving": { - "type": "trend", - "contains": "time", - "values": { - "max": 1.462209, - "avg": 0.022929372463191405, - "min": 0.006291, - "med": 0.018792, - "p(90)": 0.03475, - "p(95)": 0.042537799999999966, - "p(99)": 0.07347857999999999 - } - }, - "http_req_sending": { - "contains": "time", - "values": { - "max": 0.1985, - "avg": 0.011436388778352617, - "min": 0.002708, - "med": 0.009791, - "p(90)": 0.0155, - "p(95)": 0.018227449999999982, - "p(99)": 0.04276641999999996 - }, - "type": "trend" - }, - "http_req_failed{scenario:measure}": { - "type": "rate", - "contains": "default", - "values": { - "rate": 0, - "passes": 0, - "fails": 9001 - } - }, - "vus_max": { - "type": "gauge", - "contains": "default", - "values": { - "value": 20, - "min": 20, - "max": 20 - } - }, - "http_req_tls_handshaking": { - "type": "trend", - "contains": "time", - "values": { - "max": 0, - "avg": 0, - "min": 0, - "med": 0, - "p(90)": 0, - "p(95)": 0, - "p(99)": 0 - } - }, - "http_req_duration": { - "type": "trend", - "contains": "time", - "values": { - "max": 59.161333, - "avg": 1.6604809213091847, - "min": 0.86925, - "med": 1.35175, - "p(90)": 1.9341542000000003, - "p(95)": 2.2528895499999995, - "p(99)": 9.063710669999981 - } - }, - "http_req_duration{scenario:measure}": { - "type": "trend", - "contains": "time", - "values": { - "p(90)": 1.926416, - "p(95)": 2.214041, - "p(99)": 6.86125, - "max": 59.161333, - "avg": 1.6267745729363383, - "min": 0.897667, - "med": 1.343125 - } - }, - "dropped_iterations{scenario:measure}": { - "type": "counter", - "contains": "default", - "values": { - "count": 0, - "rate": 0 - } - }, - "http_reqs": { - "values": { - "count": 10052, - "rate": 99.95756890795728 - }, - "type": "counter", - "contains": "default" - }, - "http_req_duration{scenario:measure,name:post_message}": { - "type": "trend", - "contains": "time", - "values": { - "p(99)": 6.405855369999998, - "max": 59.161333, - "avg": 1.6004440261635895, - "min": 0.897667, - "med": 1.3235204999999999, - "p(90)": 1.9068081000000001, - "p(95)": 2.1840693499999997 - } - }, - "checks": { - "type": "rate", - "contains": "default", - "values": { - "rate": 1, - "passes": 10002, - "fails": 0 - } - }, - "vus": { - "type": "gauge", - "contains": "default", - "values": { - "value": 0, - "min": 0, - "max": 1 - } - }, - "http_req_duration{expected_response:true}": { - "type": "trend", - "contains": "time", - "values": { - "p(99)": 9.063710669999981, - "max": 59.161333, - "avg": 1.6604809213091847, - "min": 0.86925, - "med": 1.35175, - "p(90)": 1.9341542000000003, - "p(95)": 2.2528895499999995 - } - }, - "http_req_failed": { - "type": "rate", - "contains": "default", - "values": { - "rate": 0, - "passes": 0, - "fails": 10052 - } - }, - "data_sent": { - "type": "counter", - "contains": "data", - "values": { - "count": 2934144, - "rate": 29177.26831136783 - } - }, - "iterations": { - "type": "counter", - "contains": "default", - "values": { - "rate": 99.46036651585642, - "count": 10002 - } - }, - "http_req_duration{scenario:measure,name:get_history}": { - "type": "trend", - "contains": "time", - "values": { - "med": 1.416209, - "p(90)": 2.0131584, - "p(95)": 2.3368326999999995, - "p(99)": 7.265594039999997, - "max": 35.494666, - "avg": 1.7367299672225458, - "min": 1.044291 - } - }, - "data_received": { - "type": "counter", - "contains": "data", - "values": { - "count": 24412221, - "rate": 242756.29355389794 - } - }, - "dropped_iterations": { - "type": "counter", - "contains": "default", - "values": { - "count": 0, - "rate": 0 - } - }, - "http_req_blocked": { - "type": "trend", - "contains": "time", - "values": { - "min": 0.000708, - "med": 0.0015, - "p(90)": 0.002625, - "p(95)": 0.003125, - "p(99)": 0.005603579999999996, - "max": 0.456417, - "avg": 0.002264599283724603 - } + "metrics": { + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 1.780333, + "med": 3.014792, + "p(90)": 3.6596914000000003, + "p(95)": 3.9121919999999992, + "p(99)": 5.090099159999999, + "max": 56.212958, + "avg": 3.1159679878631037 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 10002, + "fails": 0 + } + }, + "iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 9001, + "rate": 89.47821589569512 + } + }, + "http_req_duration{scenario:measure,name:get_history}": { + "type": "trend", + "contains": "time", + "values": { + "max": 56.212958, + "avg": 3.303779574986465, + "min": 1.878917, + "med": 3.178666, + "p(90)": 3.8680834, + "p(95)": 4.114504299999999, + "p(99)": 5.641044819999996 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 3.6596914000000003, + "p(95)": 3.9121919999999992, + "p(99)": 5.090099159999999, + "max": 56.212958, + "avg": 3.1159679878631037, + "min": 1.780333, + "med": 3.014792 + } + }, + "http_req_duration{scenario:measure}": { + "type": "trend", + "contains": "time", + "values": { + "med": 3.01, + "p(90)": 3.639333, + "p(95)": 3.885208, + "p(99)": 4.529583, + "max": 56.212958, + "avg": 3.0670488970114356, + "min": 1.780333 + } + }, + "http_reqs{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 9001, + "rate": 89.47821589569512 + } + }, + "http_req_tls_handshaking": { + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "p(90)": 0, + "p(95)": 0, + "p(99)": 0, + "max": 0 + }, + "type": "trend" + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.022956624850776007, + "min": 0.006833, + "med": 0.020334, + "p(90)": 0.03275, + "p(95)": 0.03964344999999998, + "p(99)": 0.060725239999999965, + "max": 0.2155 + } + }, + "dropped_iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 0, + "rate": 0 + } + }, + "http_req_failed": { + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 10052 + }, + "type": "rate" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.012848683346597717, + "min": 0.003291, + "med": 0.011542, + "p(90)": 0.015667, + "p(95)": 0.018542, + "p(99)": 0.03804115999999999, + "max": 0.262875 + } + }, + "vus_max": { + "values": { + "value": 20, + "min": 20, + "max": 20 + }, + "type": "gauge", + "contains": "default" + }, + "dropped_iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 0, + "rate": 0 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0.002959, + "p(99)": 0.005936249999999989, + "max": 0.496708, + "avg": 0.0023737325905292235, + "min": 0.000708, + "med": 0.001708, + "p(90)": 0.002542 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 25528517, + "rate": 253776.930965773 + } + }, + "vus": { + "values": { + "min": 0, + "max": 1, + "value": 1 + }, + "type": "gauge", + "contains": "default" + }, + "http_req_waiting": { + "values": { + "avg": 3.0801626796657424, + "min": 1.757875, + "med": 2.9793955, + "p(90)": 3.6172958, + "p(95)": 3.8699643999999997, + "p(99)": 5.032545169999999, + "max": 56.183417 + }, + "type": "trend", + "contains": "time" + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 2921455, + "rate": 29041.94880786112 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "rate": 99.92612222903313, + "count": 10052 + } + }, + "http_req_failed{scenario:measure}": { + "values": { + "rate": 0, + "passes": 0, + "fails": 9001 + }, + "type": "rate", + "contains": "default" + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 3.101521, + "p(90)": 3.7437834000000003, + "p(95)": 3.9873521, + "p(99)": 4.69499958, + "max": 56.278375, + "avg": 3.1603651697660466, + "min": 1.826333 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 10002, + "rate": 99.42907625694284 + } + }, + "http_req_duration{scenario:measure,name:post_message}": { + "type": "trend", + "contains": "time", + "values": { + "p(99)": 4.390463, + "max": 40.37225, + "avg": 3.00593042311993, + "min": 1.780333, + "med": 2.973875, + "p(90)": 3.5802249, + "p(95)": 3.7999892499999994 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "p(99)": 0, + "max": 0.394458, + "avg": 0.0003596257461201751, + "min": 0, + "med": 0, + "p(90)": 0 + } + } } - } -} \ No newline at end of file +} diff --git a/evidence/rolling/lb_up.csv b/evidence/rolling/lb_up.csv index 203eed26..057c4d21 100644 --- a/evidence/rolling/lb_up.csv +++ b/evidence/rolling/lb_up.csv @@ -1,928 +1,936 @@ +# date: 2026-11-05T19:21:07.986361+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 t_rel_s,up --25.84,2 --25.737,2 --25.634,2 --25.525,2 --25.419,2 --25.315,2 --25.205,2 --25.091,2 --24.987,2 --24.879,2 --24.771,2 --24.66,2 --24.547,2 --24.44,2 --24.331,2 --24.217,2 --24.106,2 --23.994,2 --23.884,2 --23.777,2 --23.666,2 --23.56,2 --23.445,2 --23.333,2 --23.22,2 --23.107,2 --22.995,2 --22.882,2 --22.767,2 --22.663,2 --22.551,2 --22.447,2 --22.336,2 --22.234,2 --22.124,2 --22.012,2 --21.898,2 --21.788,2 --21.681,2 --21.569,2 --21.462,2 --21.352,2 --21.243,2 --21.129,2 --21.024,2 --20.913,2 --20.802,2 --20.688,2 --20.574,2 +-25.905,2 +-25.8,2 +-25.696,2 +-25.594,2 +-25.491,2 +-25.388,2 +-25.278,2 +-25.166,2 +-25.055,2 +-24.943,2 +-24.832,2 +-24.726,2 +-24.616,2 +-24.509,2 +-24.404,2 +-24.301,2 +-24.19,2 +-24.076,2 +-23.97,2 +-23.857,2 +-23.75,2 +-23.64,2 +-23.532,2 +-23.419,2 +-23.307,2 +-23.194,2 +-23.083,2 +-22.972,2 +-22.866,2 +-22.762,2 +-22.652,2 +-22.539,2 +-22.427,2 +-22.323,2 +-22.21,2 +-22.099,2 +-21.986,2 +-21.884,2 +-21.775,2 +-21.668,2 +-21.558,2 +-21.449,2 +-21.336,2 +-21.227,2 +-21.115,2 +-21.011,2 +-20.907,2 +-20.795,2 +-20.683,2 +-20.571,2 -20.461,2 --20.349,2 --20.241,2 --20.132,2 --20.016,2 --19.909,2 --19.802,2 --19.688,2 --19.574,2 --19.47,2 --19.359,2 --19.248,2 --19.137,2 --19.029,2 --18.919,2 --18.812,2 --18.697,2 --18.585,2 --18.474,2 --18.367,2 --18.256,2 --18.142,2 --18.03,2 --17.92,2 --17.812,2 --17.706,2 --17.602,2 --17.498,2 --17.391,2 --17.285,2 --17.173,2 --17.063,2 --16.952,2 --16.848,2 --16.741,2 --16.628,2 --16.524,2 --16.409,2 --16.301,2 --16.19,2 --16.077,2 --15.964,2 --15.855,2 --15.749,2 --15.641,2 --15.53,2 --15.417,2 --15.304,2 --15.192,2 --15.083,2 --14.969,2 --14.864,2 --14.754,2 --14.647,2 --14.543,2 --14.437,2 --14.324,2 --14.211,2 --14.104,2 --13.995,2 --13.885,2 --13.778,2 --13.67,2 --13.562,2 --13.452,2 --13.34,2 --13.226,2 --13.12,2 --13.015,2 --12.906,2 --12.803,2 --12.693,2 --12.582,2 --12.477,2 --12.373,2 --12.269,2 --12.158,2 --12.053,2 --11.945,2 --11.835,2 --11.724,2 --11.613,2 --11.501,2 --11.392,2 --11.283,2 --11.174,2 --11.067,2 --10.954,2 --10.844,2 --10.731,2 --10.621,2 --10.515,2 --10.407,2 --10.298,2 --10.185,2 --10.078,2 --9.964,2 --9.858,2 --9.751,2 --9.645,2 --9.538,2 --9.426,2 --9.311,2 --9.203,2 --9.093,2 --8.979,2 --8.868,2 --8.764,2 --8.656,2 --8.545,2 --8.439,2 --8.327,2 --8.215,2 --8.103,2 --7.999,2 --7.888,2 --7.778,2 --7.671,2 --7.56,2 --7.448,2 --7.337,2 --7.228,2 --7.125,2 --7.012,2 --6.908,2 --6.795,2 --6.685,2 --6.577,2 --6.472,2 --6.37,2 --6.261,2 --6.148,2 --6.036,2 --5.927,2 --5.813,2 --5.707,2 --5.599,2 --5.487,2 --5.376,2 --5.269,2 --5.162,2 --5.051,2 --4.945,2 --4.838,2 --4.727,2 --4.618,2 --4.507,2 --4.398,2 --4.292,2 --4.178,2 --4.071,2 --3.96,2 --3.847,2 --3.742,2 --3.63,2 --3.519,2 --3.408,2 --3.297,2 --3.191,2 --3.076,2 --2.963,2 --2.855,2 --2.745,2 --2.634,2 --2.525,2 --2.414,2 --2.301,2 --2.194,2 --2.079,2 --1.965,2 --1.856,2 --1.747,2 --1.639,2 --1.533,2 --1.42,2 --1.309,2 --1.196,2 --1.085,2 --0.982,2 --0.869,2 --0.757,2 --0.641,2 --0.528,2 --0.421,2 --0.31,2 --0.208,2 --0.104,2 --0.0,2 -0.105,2 -0.212,2 +-20.358,2 +-20.249,2 +-20.141,2 +-20.036,2 +-19.923,2 +-19.812,2 +-19.709,2 +-19.6,2 +-19.491,2 +-19.386,2 +-19.273,2 +-19.165,2 +-19.053,2 +-18.947,2 +-18.837,2 +-18.726,2 +-18.613,2 +-18.5,2 +-18.391,2 +-18.283,2 +-18.174,2 +-18.061,2 +-17.947,2 +-17.834,2 +-17.721,2 +-17.613,2 +-17.506,2 +-17.398,2 +-17.293,2 +-17.18,2 +-17.072,2 +-16.968,2 +-16.858,2 +-16.749,2 +-16.641,2 +-16.533,2 +-16.424,2 +-16.316,2 +-16.204,2 +-16.1,2 +-15.996,2 +-15.892,2 +-15.788,2 +-15.683,2 +-15.573,2 +-15.466,2 +-15.357,2 +-15.248,2 +-15.141,2 +-15.031,2 +-14.92,2 +-14.817,2 +-14.705,2 +-14.601,2 +-14.491,2 +-14.38,2 +-14.272,2 +-14.168,2 +-14.061,2 +-13.949,2 +-13.845,2 +-13.733,2 +-13.624,2 +-13.513,2 +-13.41,2 +-13.303,2 +-13.192,2 +-13.083,2 +-12.978,2 +-12.871,2 +-12.766,2 +-12.655,2 +-12.542,2 +-12.438,2 +-12.326,2 +-12.216,2 +-12.109,2 +-11.999,2 +-11.893,2 +-11.781,2 +-11.678,2 +-11.568,2 +-11.46,2 +-11.346,2 +-11.233,2 +-11.126,2 +-11.022,2 +-10.908,2 +-10.8,2 +-10.697,2 +-10.585,2 +-10.474,2 +-10.364,2 +-10.25,2 +-10.138,2 +-10.035,2 +-9.921,2 +-9.808,2 +-9.701,2 +-9.596,2 +-9.489,2 +-9.382,2 +-9.269,2 +-9.161,2 +-9.048,2 +-8.943,2 +-8.834,2 +-8.725,2 +-8.617,2 +-8.503,2 +-8.392,2 +-8.28,2 +-8.166,2 +-8.055,2 +-7.949,2 +-7.836,2 +-7.728,2 +-7.615,2 +-7.501,2 +-7.396,2 +-7.293,2 +-7.181,2 +-7.075,2 +-6.969,2 +-6.866,2 +-6.758,2 +-6.648,2 +-6.541,2 +-6.436,2 +-6.333,2 +-6.225,2 +-6.112,2 +-6.001,2 +-5.889,2 +-5.782,2 +-5.675,2 +-5.564,2 +-5.452,2 +-5.342,2 +-5.235,2 +-5.128,2 +-5.016,2 +-4.906,2 +-4.804,2 +-4.692,2 +-4.579,2 +-4.466,2 +-4.353,2 +-4.239,2 +-4.133,2 +-4.025,2 +-3.916,2 +-3.81,2 +-3.7,2 +-3.586,2 +-3.477,2 +-3.369,2 +-3.257,2 +-3.152,2 +-3.047,2 +-2.935,2 +-2.821,2 +-2.714,2 +-2.608,2 +-2.5,2 +-2.397,2 +-2.293,2 +-2.185,2 +-2.073,2 +-1.97,2 +-1.865,2 +-1.758,2 +-1.651,2 +-1.541,2 +-1.431,2 +-1.318,2 +-1.208,2 +-1.098,2 +-0.985,2 +-0.875,2 +-0.767,2 +-0.654,2 +-0.545,2 +-0.437,2 +-0.327,2 +-0.216,2 +-0.102,2 +0.01,2 +0.115,2 +0.221,2 0.323,2 -0.426,2 -0.54,2 +0.425,2 +0.534,2 0.647,2 -0.759,2 -0.863,2 -0.976,2 +0.754,2 +0.867,2 +0.973,2 1.086,2 -1.195,2 -1.303,2 -1.409,2 -1.512,2 -1.616,2 +1.197,2 +1.3,2 +1.41,2 +1.517,2 +1.62,2 1.724,2 -1.836,2 -1.947,2 -2.055,2 -2.163,2 -2.267,2 -2.371,2 -2.486,2 -2.599,2 -2.705,2 -2.809,2 -2.919,2 -3.033,2 -3.14,2 -3.246,2 -3.356,2 -3.468,2 -3.58,3 -3.693,3 -3.797,3 -3.906,3 -4.01,3 -4.12,3 -4.226,3 -4.338,3 -4.45,3 -4.556,3 -4.665,3 -4.78,3 -4.893,2 -5.003,2 -5.114,2 -5.221,2 -5.328,2 -5.441,2 -5.553,2 -5.663,2 -5.768,2 -5.879,2 -5.984,2 -6.096,2 +1.827,2 +1.932,2 +2.044,2 +2.146,2 +2.248,2 +2.35,2 +2.457,2 +2.566,2 +2.675,2 +2.781,2 +2.886,2 +3.0,2 +3.113,3 +3.215,3 +3.318,3 +3.425,3 +3.528,3 +3.641,3 +3.749,3 +3.851,3 +3.959,3 +4.065,3 +4.176,3 +4.281,2 +4.392,2 +4.499,2 +4.611,2 +4.723,2 +4.826,2 +4.937,2 +5.04,2 +5.152,2 +5.263,2 +5.374,2 +5.477,2 +5.582,2 +5.685,2 +5.789,2 +5.899,2 +6.002,2 +6.103,2 6.205,2 -6.309,2 -6.412,2 -6.515,2 -6.618,2 -6.722,2 -6.833,2 -6.937,2 -7.049,2 -7.161,2 -7.269,2 -7.382,2 -7.489,2 -7.6,2 -7.703,2 -7.809,2 -7.915,2 -8.023,2 -8.135,2 -8.243,2 -8.347,2 -8.458,2 -8.569,2 -8.683,2 -8.792,2 -8.903,2 -9.016,2 -9.13,2 -9.238,2 -9.345,2 -9.447,3 -9.559,3 -9.671,3 -9.785,3 -9.894,3 -9.999,3 -10.108,3 -10.218,3 -10.326,3 -10.438,3 -10.551,3 -10.663,2 -10.77,2 -10.882,2 -10.986,2 -11.094,2 -11.203,2 -11.307,2 -11.417,2 -11.525,2 -11.629,2 -11.743,2 -11.855,2 -11.963,2 -12.07,2 -12.185,2 -12.291,2 -12.404,2 -12.511,2 -12.614,2 -12.724,2 -12.834,2 -12.946,2 -13.056,2 -13.165,2 -13.277,2 -13.385,2 -13.498,2 -13.606,2 -13.717,2 -13.821,2 -13.933,2 -14.041,2 -14.147,2 -14.259,2 -14.364,2 -14.477,2 -14.591,2 -14.698,2 -14.803,2 -14.915,2 -15.028,2 -15.141,2 -15.253,2 -15.362,2 -15.467,2 -15.571,2 -15.684,2 -15.791,2 -15.903,2 -16.018,2 -16.131,2 -16.244,2 -16.357,2 -16.46,2 -16.571,2 -16.684,2 -16.798,2 -16.904,2 -17.018,2 -17.129,2 -17.234,2 -17.341,2 -17.447,2 -17.557,2 -17.666,2 -17.774,2 -17.886,2 -17.999,2 -18.109,2 -18.215,2 -18.329,2 -18.438,2 -18.542,2 -18.65,2 -18.755,2 -18.864,2 -18.968,2 -19.079,2 -19.188,2 -19.3,2 -19.404,2 -19.517,2 -19.632,2 -19.746,2 -19.855,2 -19.976,2 -20.115,2 -20.234,2 -20.361,2 -20.478,2 -20.605,2 -20.725,2 -20.87,2 -20.984,2 -21.101,2 -21.213,2 -21.317,2 -21.426,2 -21.54,2 -21.651,2 -21.755,2 -21.865,2 -21.97,2 -22.083,2 -22.195,2 -22.304,2 -22.416,2 -22.527,2 -22.64,2 -22.751,2 -22.864,2 -22.977,2 -23.085,2 -23.197,2 -23.308,2 -23.42,2 -23.531,2 -23.641,2 -23.752,2 -23.861,2 -23.971,2 -24.083,2 -24.195,2 -24.309,2 -24.417,2 -24.527,2 -24.634,2 -24.748,2 -24.86,2 -24.968,2 -25.077,2 -25.187,2 -25.294,2 -25.397,2 -25.506,2 -25.618,2 -25.724,2 +6.317,2 +6.425,2 +6.532,2 +6.641,2 +6.751,2 +6.854,2 +6.961,2 +7.068,2 +7.176,2 +7.286,2 +7.389,2 +7.499,2 +7.609,2 +7.72,2 +7.832,3 +7.941,3 +8.045,3 +8.152,3 +8.262,3 +8.376,3 +8.488,3 +8.592,3 +8.7,3 +8.81,2 +8.914,2 +9.025,2 +9.137,2 +9.249,2 +9.363,2 +9.473,2 +9.578,2 +9.682,2 +9.79,2 +9.898,2 +10.003,2 +10.108,2 +10.212,2 +10.324,2 +10.435,2 +10.54,2 +10.643,2 +10.748,2 +10.852,2 +10.958,2 +11.062,2 +11.172,2 +11.283,2 +11.388,2 +11.501,2 +11.608,2 +11.721,2 +11.833,2 +11.942,2 +12.05,2 +12.157,2 +12.261,2 +12.371,2 +12.476,2 +12.583,2 +12.69,2 +12.804,2 +12.917,2 +13.023,2 +13.131,2 +13.244,2 +13.349,2 +13.458,2 +13.567,2 +13.676,2 +13.783,2 +13.894,2 +14.003,2 +14.109,2 +14.217,2 +14.327,2 +14.433,2 +14.546,2 +14.651,2 +14.761,2 +14.869,2 +14.974,2 +15.083,2 +15.197,2 +15.309,2 +15.418,2 +15.531,2 +15.64,2 +15.752,2 +15.857,2 +15.97,2 +16.082,2 +16.191,2 +16.305,2 +16.416,2 +16.521,2 +16.633,2 +16.743,2 +16.849,2 +16.953,2 +17.061,2 +17.165,2 +17.276,2 +17.39,2 +17.493,2 +17.603,2 +17.706,2 +17.821,2 +17.932,2 +18.036,2 +18.14,2 +18.249,2 +18.36,2 +18.463,2 +18.575,2 +18.678,2 +18.781,2 +18.891,2 +19.0,2 +19.107,2 +19.217,2 +19.321,2 +19.424,2 +19.537,2 +19.649,2 +19.754,2 +19.861,2 +19.966,2 +20.071,2 +20.182,2 +20.295,2 +20.408,2 +20.516,2 +20.628,2 +20.732,2 +20.836,2 +20.948,2 +21.055,2 +21.158,2 +21.262,2 +21.374,2 +21.48,2 +21.591,2 +21.698,2 +21.808,2 +21.916,2 +22.02,2 +22.132,2 +22.235,2 +22.347,2 +22.457,2 +22.569,2 +22.683,2 +22.797,2 +22.905,2 +23.014,2 +23.124,2 +23.23,2 +23.341,2 +23.445,2 +23.558,2 +23.67,2 +23.775,2 +23.888,2 +23.999,2 +24.11,2 +24.216,2 +24.326,2 +24.431,2 +24.534,2 +24.643,2 +24.751,2 +24.858,2 +24.969,2 +25.079,2 +25.19,2 +25.299,2 +25.404,2 +25.51,2 +25.619,2 +25.721,2 25.828,2 -25.936,2 -26.048,2 -26.156,2 -26.264,2 -26.369,2 -26.479,2 -26.585,2 -26.698,2 -26.81,2 -26.925,2 -27.036,2 -27.141,2 -27.248,2 -27.36,2 -27.465,2 -27.578,2 -27.68,2 -27.788,2 -27.9,2 -28.01,2 -28.125,2 -28.232,2 -28.339,2 +25.94,2 +26.055,2 +26.166,2 +26.272,2 +26.381,2 +26.494,2 +26.599,2 +26.703,2 +26.806,2 +26.918,2 +27.026,2 +27.133,2 +27.245,2 +27.357,2 +27.468,2 +27.58,2 +27.687,2 +27.792,2 +27.904,2 +28.017,2 +28.131,2 +28.243,2 +28.346,2 28.449,2 -28.562,2 -28.669,2 -28.78,2 -28.883,2 -28.994,2 -29.101,2 -29.214,2 -29.324,2 -29.439,2 -29.546,2 -29.651,2 -29.755,2 -29.866,2 -29.987,2 -30.098,2 -30.207,2 -30.32,2 -30.433,2 -30.545,2 -30.656,2 -30.763,2 -30.873,2 -30.985,2 -31.09,2 -31.205,2 -31.309,2 -31.423,2 -31.532,2 -31.638,2 -31.75,2 -31.861,2 -31.969,2 -32.08,2 -32.194,2 -32.304,2 -32.415,2 -32.527,2 -32.632,2 -32.743,2 -32.856,2 -32.963,2 -33.075,2 -33.184,2 -33.288,2 -33.399,2 -33.511,2 -33.625,2 -33.731,2 -33.84,2 -33.947,2 -34.058,2 -34.162,2 -34.266,2 -34.379,2 -34.487,2 -34.591,2 -34.697,2 -34.802,2 -34.913,2 -35.025,2 -35.137,2 -35.241,2 -35.352,2 -35.462,2 -35.569,2 -35.675,2 -35.778,2 -35.891,2 -36.0,2 -36.113,2 -36.226,2 -36.336,2 -36.448,2 -36.562,2 -36.674,2 -36.778,2 -36.888,2 -37.0,2 -37.114,2 -37.22,2 -37.334,2 -37.446,2 -37.55,2 -37.654,2 -37.758,2 -37.864,2 -37.97,2 -38.083,2 -38.191,2 -38.294,2 -38.401,2 -38.51,2 -38.615,2 -38.717,2 -38.829,2 -38.932,2 -39.036,2 -39.144,2 -39.255,2 -39.367,2 -39.478,2 -39.585,2 -39.697,2 -39.805,2 -39.91,2 -40.016,2 -40.124,2 -40.237,2 -40.343,2 -40.447,2 -40.554,2 -40.664,2 -40.767,2 -40.875,2 -40.976,2 -41.086,2 -41.189,2 -41.292,2 -41.401,2 -41.514,2 -41.62,2 -41.731,2 -41.844,2 -41.953,2 -42.066,2 -42.173,2 -42.28,2 -42.393,2 -42.5,2 -42.612,2 -42.726,2 -42.835,2 -42.95,2 -43.057,2 -43.165,2 -43.269,2 -43.375,2 -43.488,2 -43.595,2 -43.699,2 -43.81,2 -43.924,2 -44.035,2 -44.141,2 -44.248,2 -44.358,2 -44.472,2 -44.586,2 -44.696,2 -44.806,2 -44.913,2 -45.022,2 -45.127,2 -45.24,2 -45.348,2 -45.463,2 -45.578,2 -45.692,2 -45.799,2 -45.906,2 -46.01,2 -46.123,2 -46.236,2 -46.347,2 -46.452,2 -46.558,2 -46.666,2 -46.779,2 -46.883,2 -46.994,2 -47.103,2 -47.217,2 -47.331,2 -47.444,2 -47.555,2 -47.664,2 -47.776,2 -47.878,2 -47.989,2 -48.101,2 -48.215,2 -48.319,2 -48.426,2 -48.531,2 -48.641,2 -48.755,2 -48.866,2 -48.979,2 -49.088,2 -49.2,2 -49.31,2 -49.421,2 -49.53,2 -49.641,2 -49.749,2 -49.859,2 -49.966,2 -50.074,2 -50.178,2 -50.285,2 -50.389,2 -50.493,2 -50.603,2 -50.716,2 -50.822,2 -50.925,2 -51.038,2 -51.142,2 -51.247,2 -51.362,2 -51.466,2 -51.576,2 -51.69,2 -51.797,2 -51.905,2 -52.011,2 -52.123,2 -52.234,2 -52.347,2 -52.454,2 -52.566,2 -52.68,2 -52.785,2 -52.891,2 -52.998,2 -53.108,2 -53.223,2 -53.329,2 -53.441,2 -53.554,2 -53.666,2 -53.77,2 -53.879,2 -53.987,2 -54.098,2 -54.209,2 -54.318,2 -54.428,2 -54.543,2 -54.656,2 -54.768,2 -54.881,2 -54.993,2 -55.105,2 -55.215,2 -55.322,2 -55.435,2 -55.537,2 -55.646,2 -55.753,2 -55.861,2 -55.975,2 -56.08,2 -56.189,2 -56.295,2 -56.408,2 -56.516,2 -56.627,2 -56.74,2 -56.85,2 -56.956,2 -57.069,2 -57.179,2 -57.293,2 -57.404,2 -57.513,2 -57.621,2 -57.732,2 -57.834,2 -57.947,2 -58.058,2 -58.169,2 -58.274,2 -58.38,2 -58.492,2 -58.599,2 -58.705,2 -58.812,2 -58.921,2 -59.034,2 -59.138,2 -59.244,2 -59.352,2 -59.465,2 -59.577,2 -59.685,2 -59.792,2 +28.553,2 +28.655,2 +28.766,2 +28.873,2 +28.986,2 +29.098,2 +29.206,2 +29.317,2 +29.425,2 +29.53,2 +29.642,2 +29.749,2 +29.862,2 +29.972,2 +30.084,2 +30.189,2 +30.301,2 +30.407,2 +30.514,2 +30.624,2 +30.732,2 +30.844,2 +30.957,2 +31.071,2 +31.183,2 +31.286,2 +31.397,2 +31.507,2 +31.611,2 +31.723,2 +31.836,2 +31.948,2 +32.052,2 +32.163,2 +32.268,2 +32.374,2 +32.478,2 +32.581,2 +32.688,2 +32.798,2 +32.907,2 +33.019,2 +33.126,2 +33.239,2 +33.347,2 +33.453,2 +33.563,2 +33.674,2 +33.778,2 +33.882,2 +33.986,2 +34.093,2 +34.2,2 +34.307,2 +34.421,2 +34.53,2 +34.643,2 +34.75,2 +34.864,2 +34.975,2 +35.088,2 +35.197,2 +35.311,2 +35.423,2 +35.532,2 +35.64,2 +35.749,2 +35.86,2 +35.968,2 +36.081,2 +36.191,2 +36.303,2 +36.407,2 +36.512,2 +36.626,2 +36.738,2 +36.847,2 +36.959,2 +37.068,2 +37.18,2 +37.283,2 +37.395,2 +37.5,2 +37.603,2 +37.713,2 +37.824,2 +37.938,2 +38.044,2 +38.157,2 +38.263,2 +38.372,2 +38.477,2 +38.588,2 +38.7,2 +38.803,2 +38.907,2 +39.015,2 +39.124,2 +39.231,2 +39.344,2 +39.457,2 +39.57,2 +39.672,2 +39.779,2 +39.892,2 +39.997,2 +40.107,2 +40.213,2 +40.316,2 +40.424,2 +40.527,2 +40.632,2 +40.744,2 +40.849,2 +40.959,2 +41.067,2 +41.174,2 +41.281,2 +41.393,2 +41.498,2 +41.602,2 +41.707,2 +41.814,2 +41.921,2 +42.024,2 +42.136,2 +42.249,2 +42.355,2 +42.462,2 +42.575,2 +42.685,2 +42.798,2 +42.911,2 +43.025,2 +43.128,2 +43.238,2 +43.346,2 +43.453,2 +43.559,2 +43.667,2 +43.776,2 +43.889,2 +43.997,2 +44.105,2 +44.215,2 +44.327,2 +44.432,2 +44.546,2 +44.657,2 +44.769,2 +44.874,2 +44.985,2 +45.094,2 +45.207,2 +45.313,2 +45.425,2 +45.537,2 +45.649,2 +45.761,2 +45.87,2 +45.974,2 +46.085,2 +46.189,2 +46.298,2 +46.407,2 +46.514,2 +46.619,2 +46.731,2 +46.839,2 +46.95,2 +47.06,2 +47.164,2 +47.274,2 +47.384,2 +47.488,2 +47.599,2 +47.707,2 +47.814,2 +47.922,2 +48.031,2 +48.136,2 +48.249,2 +48.36,2 +48.471,2 +48.577,2 +48.69,2 +48.792,2 +48.898,2 +49.008,2 +49.116,2 +49.226,2 +49.334,2 +49.44,2 +49.546,2 +49.657,2 +49.764,2 +49.873,2 +49.981,2 +50.086,2 +50.188,2 +50.297,2 +50.405,2 +50.515,2 +50.624,2 +50.731,2 +50.844,2 +50.955,2 +51.065,2 +51.175,2 +51.287,2 +51.399,2 +51.506,2 +51.618,2 +51.728,2 +51.84,2 +51.945,2 +52.048,2 +52.156,2 +52.266,2 +52.378,2 +52.491,2 +52.603,2 +52.715,2 +52.823,2 +52.935,2 +53.04,2 +53.147,2 +53.26,2 +53.369,2 +53.476,2 +53.581,2 +53.692,2 +53.798,2 +53.911,2 +54.024,2 +54.137,2 +54.244,2 +54.351,2 +54.457,2 +54.563,2 +54.673,2 +54.781,2 +54.892,2 +55.0,2 +55.113,2 +55.226,2 +55.339,2 +55.443,2 +55.556,2 +55.665,2 +55.778,2 +55.889,2 +55.991,2 +56.094,2 +56.202,2 +56.31,2 +56.417,2 +56.525,2 +56.638,2 +56.748,2 +56.861,2 +56.972,2 +57.08,2 +57.192,2 +57.304,2 +57.416,2 +57.528,2 +57.638,2 +57.743,2 +57.851,2 +57.958,2 +58.067,2 +58.172,2 +58.283,2 +58.388,2 +58.495,2 +58.606,2 +58.712,2 +58.823,2 +58.936,2 +59.048,2 +59.157,2 +59.264,2 +59.371,2 +59.476,2 +59.588,2 +59.692,2 +59.798,2 59.903,2 -60.01,2 -60.117,2 -60.228,2 -60.332,2 -60.438,2 -60.552,2 -60.661,2 -60.77,2 -60.881,2 -60.986,2 -61.092,2 -61.2,2 -61.311,2 -61.417,2 -61.525,2 -61.638,2 -61.744,2 -61.856,2 -61.964,2 -62.071,2 -62.178,2 -62.283,2 -62.39,2 -62.501,2 -62.609,2 -62.714,2 -62.824,2 -62.93,2 -63.034,2 -63.145,2 -63.248,2 -63.363,2 -63.473,2 -63.576,2 -63.688,2 -63.796,2 -63.902,2 -64.014,2 -64.125,2 -64.236,2 -64.341,2 -64.448,2 -64.559,2 -64.664,2 -64.774,2 -64.882,2 -64.986,2 -65.096,2 -65.203,2 -65.311,2 -65.423,2 -65.53,2 -65.642,2 -65.754,2 -65.864,2 -65.97,2 -66.082,2 -66.193,2 -66.304,2 -66.409,2 -66.521,2 -66.634,2 -66.739,2 -66.846,2 -66.959,2 -67.067,2 -67.171,2 -67.276,2 -67.39,2 -67.497,2 -67.604,2 -67.718,2 -67.825,2 -67.935,2 -68.047,2 -68.158,2 -68.27,2 -68.375,2 -68.483,2 -68.597,2 -68.706,2 -68.816,2 -68.919,2 -69.025,2 -69.128,2 -69.237,2 -69.352,2 -69.459,2 -69.572,2 -69.682,2 -69.789,2 -69.895,2 -70.007,2 -70.12,2 -70.232,2 -70.346,2 -70.461,2 -70.571,2 -70.684,2 -70.789,2 -70.894,2 -71.001,2 -71.115,2 -71.225,2 -71.338,2 -71.448,2 -71.557,2 -71.663,2 -71.772,2 -71.876,2 -71.988,2 -72.093,2 -72.205,2 -72.309,2 -72.416,2 -72.522,2 -72.634,2 -72.745,2 -72.858,2 -72.962,2 -73.073,2 -73.186,2 -73.298,2 -73.409,2 -73.516,2 -73.626,2 -73.737,2 -73.844,2 -73.947,2 -74.057,2 -74.16,2 -74.271,2 +60.017,2 +60.125,2 +60.238,2 +60.345,2 +60.456,2 +60.567,2 +60.68,2 +60.791,2 +60.903,2 +61.011,2 +61.114,2 +61.225,2 +61.338,2 +61.448,2 +61.556,2 +61.659,2 +61.766,2 +61.878,2 +61.99,2 +62.096,2 +62.206,2 +62.319,2 +62.421,2 +62.533,2 +62.64,2 +62.747,2 +62.855,2 +62.964,2 +63.066,2 +63.176,2 +63.287,2 +63.397,2 +63.508,2 +63.62,2 +63.723,2 +63.83,2 +63.942,2 +64.048,2 +64.156,2 +64.263,2 +64.365,2 +64.476,2 +64.588,2 +64.697,2 +64.803,2 +64.916,2 +65.027,2 +65.137,2 +65.247,2 +65.357,2 +65.463,2 +65.569,2 +65.679,2 +65.789,2 +65.903,2 +66.015,2 +66.129,2 +66.233,2 +66.347,2 +66.458,2 +66.563,2 +66.675,2 +66.786,2 +66.898,2 +67.006,2 +67.113,2 +67.223,2 +67.334,2 +67.439,2 +67.543,2 +67.648,2 +67.754,2 +67.86,2 +67.962,2 +68.065,2 +68.167,2 +68.274,2 +68.388,2 +68.499,2 +68.612,2 +68.724,2 +68.827,2 +68.931,2 +69.043,2 +69.156,2 +69.261,2 +69.368,2 +69.475,2 +69.577,2 +69.685,2 +69.793,2 +69.906,2 +70.016,2 +70.126,2 +70.237,2 +70.347,2 +70.459,2 +70.566,2 +70.674,2 +70.783,2 +70.885,2 +70.997,2 +71.109,2 +71.223,2 +71.329,2 +71.441,2 +71.549,2 +71.652,2 +71.756,2 +71.869,2 +71.98,2 +72.092,2 +72.201,2 +72.31,2 +72.422,2 +72.534,2 +72.645,2 +72.758,2 +72.862,2 +72.972,2 +73.074,2 +73.181,2 +73.291,2 +73.397,2 +73.508,2 +73.615,2 +73.728,2 +73.834,2 +73.948,2 +74.055,2 +74.164,2 +74.269,2 74.375,2 -74.484,2 -74.597,2 -74.704,2 -74.81,2 -74.923,2 -75.028,2 -75.136,2 -75.249,2 -75.359,2 -75.463,2 +74.478,2 +74.587,2 +74.689,2 +74.796,2 +74.9,2 +75.008,2 +75.112,2 +75.222,2 +75.338,2 +75.447,2 diff --git a/evidence/rolling/probe.csv b/evidence/rolling/probe.csv index 44bb9223..0bb3208b 100644 --- a/evidence/rolling/probe.csv +++ b/evidence/rolling/probe.csv @@ -1,920 +1,915 @@ +# date: 2026-11-05T19:21:07.986361+00:00 commit: 252e3e3ffeea5004cfd9dc342e686af1417a77a2 t_rel_s,status,latency_ms,instance --25.84,201,4.3,6c2a02267340 --25.735,201,4.0,158dfc1bd95d --25.631,201,3.4,6c2a02267340 --25.525,201,5.8,158dfc1bd95d --25.413,201,4.1,6c2a02267340 --25.302,201,4.3,158dfc1bd95d --25.188,201,5.0,158dfc1bd95d --25.075,201,6.7,6c2a02267340 --24.959,201,3.7,158dfc1bd95d --24.847,201,3.7,6c2a02267340 --24.733,201,4.4,6c2a02267340 --24.628,201,5.8,6c2a02267340 --24.512,201,5.0,6c2a02267340 --24.405,201,4.1,6c2a02267340 --24.295,201,5.0,6c2a02267340 --24.182,201,3.9,6c2a02267340 --24.07,201,3.0,6c2a02267340 --23.959,201,4.3,158dfc1bd95d --23.854,201,3.8,6c2a02267340 --23.74,201,4.9,6c2a02267340 --23.627,201,4.3,6c2a02267340 --23.513,201,5.4,158dfc1bd95d --23.406,201,4.8,158dfc1bd95d --23.294,201,4.3,158dfc1bd95d --23.183,201,3.2,158dfc1bd95d --23.074,201,2.8,158dfc1bd95d --22.961,201,3.4,158dfc1bd95d --22.85,201,4.6,158dfc1bd95d --22.737,201,3.4,6c2a02267340 --22.625,201,5.1,6c2a02267340 --22.513,201,3.8,6c2a02267340 --22.405,201,4.3,6c2a02267340 --22.299,201,3.4,6c2a02267340 --22.186,201,3.5,6c2a02267340 --22.078,201,4.5,6c2a02267340 --21.963,201,5.6,6c2a02267340 --21.851,201,4.4,6c2a02267340 --21.743,201,5.1,6c2a02267340 --21.629,201,4.7,158dfc1bd95d --21.524,201,5.7,6c2a02267340 --21.416,201,4.2,6c2a02267340 --21.303,201,3.4,6c2a02267340 --21.193,201,3.6,6c2a02267340 --21.084,201,5.6,6c2a02267340 --20.974,201,5.5,6c2a02267340 --20.858,201,6.9,158dfc1bd95d --20.743,201,5.0,158dfc1bd95d --20.628,201,5.2,6c2a02267340 --20.517,201,3.8,6c2a02267340 --20.409,201,3.4,158dfc1bd95d --20.297,201,4.0,6c2a02267340 --20.184,201,6.9,6c2a02267340 --20.071,201,5.3,6c2a02267340 --19.964,201,4.9,6c2a02267340 --19.849,201,6.1,158dfc1bd95d --19.737,201,5.0,158dfc1bd95d --19.63,201,4.6,6c2a02267340 --19.523,201,3.6,6c2a02267340 --19.411,201,3.4,6c2a02267340 --19.298,201,4.8,158dfc1bd95d --19.183,201,4.4,158dfc1bd95d --19.073,201,4.4,158dfc1bd95d --18.966,201,4.4,158dfc1bd95d --18.856,201,5.2,158dfc1bd95d --18.741,201,5.3,158dfc1bd95d --18.629,201,3.3,158dfc1bd95d --18.524,201,4.6,158dfc1bd95d --18.416,201,4.2,158dfc1bd95d --18.306,201,4.0,158dfc1bd95d --18.194,201,4.5,158dfc1bd95d --18.085,201,4.7,158dfc1bd95d --17.97,201,4.2,158dfc1bd95d --17.859,201,5.3,6c2a02267340 --17.743,201,2.9,6c2a02267340 --17.64,201,4.4,158dfc1bd95d --17.528,201,4.2,6c2a02267340 --17.417,201,5.1,6c2a02267340 --17.307,201,4.1,6c2a02267340 --17.196,201,4.1,6c2a02267340 --17.087,201,3.7,6c2a02267340 --16.976,201,4.8,6c2a02267340 --16.867,201,3.9,6c2a02267340 --16.763,201,3.2,158dfc1bd95d --16.654,201,4.4,158dfc1bd95d --16.543,201,4.9,158dfc1bd95d --16.428,201,5.1,6c2a02267340 --16.313,201,4.9,6c2a02267340 --16.2,201,5.7,158dfc1bd95d --16.093,201,3.3,6c2a02267340 --15.981,201,3.8,6c2a02267340 --15.874,201,4.5,6c2a02267340 --15.768,201,4.6,6c2a02267340 --15.659,201,7.0,6c2a02267340 --15.552,201,4.8,158dfc1bd95d --15.446,201,4.8,158dfc1bd95d --15.336,201,4.9,158dfc1bd95d --15.231,201,5.8,6c2a02267340 --15.119,201,5.8,158dfc1bd95d --15.003,201,6.2,6c2a02267340 --14.895,201,4.9,6c2a02267340 --14.783,201,3.7,6c2a02267340 --14.669,201,6.8,158dfc1bd95d --14.56,201,4.8,6c2a02267340 --14.452,201,6.4,6c2a02267340 --14.34,201,6.0,6c2a02267340 --14.224,201,4.3,158dfc1bd95d --14.117,201,5.4,6c2a02267340 --14.004,201,4.6,158dfc1bd95d --13.895,201,3.8,158dfc1bd95d --13.781,201,4.7,158dfc1bd95d --13.669,201,5.0,6c2a02267340 --13.564,201,5.6,158dfc1bd95d --13.453,201,3.9,158dfc1bd95d --13.339,201,5.0,6c2a02267340 --13.226,201,3.3,6c2a02267340 --13.116,201,4.1,6c2a02267340 --13.006,201,3.9,6c2a02267340 --12.901,201,2.9,158dfc1bd95d --12.794,201,3.8,158dfc1bd95d --12.689,201,4.3,6c2a02267340 --12.575,201,2.5,158dfc1bd95d --12.464,201,2.9,158dfc1bd95d --12.354,201,3.8,158dfc1bd95d --12.246,201,4.1,158dfc1bd95d --12.14,201,4.2,6c2a02267340 --12.033,201,3.3,6c2a02267340 --11.92,201,4.5,6c2a02267340 --11.806,201,5.5,158dfc1bd95d --11.69,201,3.1,158dfc1bd95d --11.581,201,2.6,158dfc1bd95d --11.476,201,3.1,158dfc1bd95d --11.366,201,5.0,158dfc1bd95d --11.259,201,4.3,158dfc1bd95d --11.144,201,3.3,158dfc1bd95d --11.033,201,3.7,158dfc1bd95d --10.924,201,5.5,158dfc1bd95d --10.81,201,5.1,158dfc1bd95d --10.703,201,5.2,158dfc1bd95d --10.588,201,5.9,158dfc1bd95d --10.477,201,5.3,6c2a02267340 --10.367,201,4.6,6c2a02267340 --10.254,201,6.2,6c2a02267340 --10.142,201,6.4,6c2a02267340 --10.03,201,5.9,158dfc1bd95d --9.914,201,8.2,158dfc1bd95d --9.803,201,5.0,158dfc1bd95d --9.693,201,5.0,158dfc1bd95d --9.581,201,5.8,158dfc1bd95d --9.472,201,4.8,158dfc1bd95d --9.359,201,5.0,158dfc1bd95d --9.251,201,4.6,158dfc1bd95d --9.143,201,3.3,158dfc1bd95d --9.038,201,3.3,6c2a02267340 --8.934,201,4.7,6c2a02267340 --8.82,201,5.3,158dfc1bd95d --8.706,201,7.8,158dfc1bd95d --8.594,201,4.0,158dfc1bd95d --8.483,201,5.4,158dfc1bd95d --8.37,201,5.1,158dfc1bd95d --8.261,201,5.6,158dfc1bd95d --8.149,201,3.2,158dfc1bd95d --8.04,201,3.1,158dfc1bd95d --7.927,201,5.3,6c2a02267340 --7.82,201,3.9,158dfc1bd95d --7.713,201,4.7,158dfc1bd95d --7.608,201,4.7,158dfc1bd95d --7.503,201,3.8,6c2a02267340 --7.398,201,3.5,6c2a02267340 --7.291,201,4.6,158dfc1bd95d --7.185,201,5.4,158dfc1bd95d --7.075,201,4.4,158dfc1bd95d --6.97,201,4.9,158dfc1bd95d --6.859,201,4.4,158dfc1bd95d --6.746,201,4.5,158dfc1bd95d --6.632,201,4.0,158dfc1bd95d --6.525,201,3.5,158dfc1bd95d --6.42,201,3.6,6c2a02267340 --6.313,201,3.5,6c2a02267340 --6.208,201,5.1,6c2a02267340 --6.093,201,5.2,6c2a02267340 --5.979,201,5.2,6c2a02267340 --5.872,201,5.1,6c2a02267340 --5.761,201,4.9,6c2a02267340 --5.651,201,4.3,6c2a02267340 --5.542,201,3.7,6c2a02267340 --5.434,201,3.0,6c2a02267340 --5.33,201,5.4,6c2a02267340 --5.215,201,5.2,6c2a02267340 --5.103,201,4.5,6c2a02267340 --4.996,201,4.4,6c2a02267340 --4.887,201,4.5,6c2a02267340 --4.777,201,4.8,6c2a02267340 --4.668,201,4.0,6c2a02267340 --4.556,201,3.6,6c2a02267340 --4.444,201,6.1,6c2a02267340 --4.335,201,7.3,6c2a02267340 --4.224,201,5.6,6c2a02267340 --4.115,201,5.7,6c2a02267340 --4.006,201,6.2,6c2a02267340 --3.895,201,5.0,6c2a02267340 --3.788,201,3.1,158dfc1bd95d --3.674,201,3.2,6c2a02267340 --3.567,201,3.6,6c2a02267340 --3.462,201,4.0,158dfc1bd95d --3.355,201,4.7,158dfc1bd95d --3.242,201,5.6,158dfc1bd95d --3.135,201,3.8,158dfc1bd95d --3.023,201,4.9,158dfc1bd95d --2.913,201,4.3,158dfc1bd95d --2.8,201,3.5,158dfc1bd95d --2.693,201,3.8,158dfc1bd95d --2.585,201,5.0,158dfc1bd95d --2.478,201,4.2,158dfc1bd95d --2.367,201,4.0,158dfc1bd95d --2.253,201,5.4,158dfc1bd95d --2.146,201,4.6,158dfc1bd95d --2.033,201,4.1,158dfc1bd95d --1.923,201,3.2,158dfc1bd95d --1.815,201,3.0,158dfc1bd95d --1.702,201,4.5,158dfc1bd95d --1.59,201,4.8,158dfc1bd95d --1.485,201,3.6,158dfc1bd95d --1.381,201,4.0,6c2a02267340 --1.266,201,3.9,158dfc1bd95d --1.153,201,5.2,158dfc1bd95d --1.047,201,3.2,158dfc1bd95d --0.934,201,3.6,158dfc1bd95d --0.821,201,4.8,158dfc1bd95d --0.712,201,4.9,158dfc1bd95d --0.603,201,5.3,158dfc1bd95d --0.494,201,3.6,158dfc1bd95d --0.39,201,3.9,6c2a02267340 --0.276,201,3.7,158dfc1bd95d --0.165,201,9.0,158dfc1bd95d --0.053,201,3.4,158dfc1bd95d -0.057,201,4.7,158dfc1bd95d -0.163,201,3.6,158dfc1bd95d -0.276,201,7.2,158dfc1bd95d -0.384,201,3.8,158dfc1bd95d -0.488,201,3.6,6c2a02267340 -0.593,201,4.3,158dfc1bd95d -0.699,201,3.9,158dfc1bd95d -0.804,201,5.1,158dfc1bd95d -0.915,201,8.1,158dfc1bd95d -1.029,201,3.9,158dfc1bd95d -1.139,201,4.1,158dfc1bd95d -1.252,201,4.5,6c2a02267340 -1.359,201,4.0,158dfc1bd95d -1.463,201,4.2,158dfc1bd95d -1.569,201,4.9,6c2a02267340 -1.678,201,4.9,6c2a02267340 -1.792,201,4.4,158dfc1bd95d -1.897,201,7.5,6c2a02267340 -2.012,201,5.3,158dfc1bd95d -2.119,201,4.6,6c2a02267340 -2.226,201,3.6,6c2a02267340 -2.332,201,3.9,158dfc1bd95d -2.441,201,4.4,158dfc1bd95d -2.554,201,4.0,6c2a02267340 -2.668,201,3.7,6c2a02267340 -2.776,201,3.6,6c2a02267340 -2.884,201,5.1,6c2a02267340 -2.993,201,3.4,6c2a02267340 -3.101,201,3.9,6c2a02267340 -3.207,201,4.8,158dfc1bd95d -3.321,201,4.6,6c2a02267340 -3.429,201,9.7,158dfc1bd95d -3.547,201,6.4,6c2a02267340 -3.656,201,5.6,6c2a02267340 -3.766,201,4.0,6c2a02267340 -3.877,201,5.2,6c2a02267340 -3.992,201,4.1,158dfc1bd95d -4.104,201,4.5,158dfc1bd95d -4.218,201,3.8,158dfc1bd95d -4.327,201,4.0,158dfc1bd95d -4.441,201,4.5,158dfc1bd95d -4.553,201,4.9,c2f5d82dee7c -4.665,201,5.9,c2f5d82dee7c -4.78,201,4.1,c2f5d82dee7c -4.892,201,3.5,c2f5d82dee7c -4.999,201,3.0,c2f5d82dee7c -5.11,201,5.6,c2f5d82dee7c -5.217,201,4.3,c2f5d82dee7c -5.324,201,4.7,c2f5d82dee7c -5.436,201,4.5,c2f5d82dee7c -5.545,201,3.9,c2f5d82dee7c -5.654,201,3.8,c2f5d82dee7c -5.765,201,4.8,c2f5d82dee7c -5.88,201,3.3,c2f5d82dee7c -5.986,201,2.4,c2f5d82dee7c -6.096,201,4.8,c2f5d82dee7c -6.21,201,4.4,c2f5d82dee7c -6.32,201,3.9,c2f5d82dee7c -6.425,201,3.4,c2f5d82dee7c -6.528,201,3.5,158dfc1bd95d -6.633,201,3.8,158dfc1bd95d -6.74,201,3.4,c2f5d82dee7c -6.846,201,4.3,c2f5d82dee7c -6.954,201,3.6,c2f5d82dee7c -7.065,201,3.9,c2f5d82dee7c -7.176,201,4.1,c2f5d82dee7c -7.289,201,4.0,c2f5d82dee7c -7.401,201,6.0,158dfc1bd95d -7.517,201,3.6,158dfc1bd95d -7.629,201,2.9,158dfc1bd95d -7.736,201,3.9,158dfc1bd95d -7.845,201,4.2,158dfc1bd95d -7.956,201,5.1,158dfc1bd95d -8.071,201,4.3,c2f5d82dee7c -8.186,201,5.7,c2f5d82dee7c -8.292,201,3.6,c2f5d82dee7c -8.406,201,4.9,c2f5d82dee7c -8.517,201,2.9,c2f5d82dee7c -8.63,201,5.6,c2f5d82dee7c -8.742,201,4.4,158dfc1bd95d -8.857,201,5.4,158dfc1bd95d -8.972,201,4.7,c2f5d82dee7c -9.081,201,4.5,c2f5d82dee7c -9.19,201,4.2,158dfc1bd95d -9.297,201,4.8,158dfc1bd95d -9.405,201,3.5,158dfc1bd95d -9.519,201,5.6,65d10d346f4e -9.627,201,5.4,65d10d346f4e -9.734,201,4.5,65d10d346f4e -9.843,201,6.2,65d10d346f4e -9.959,201,5.1,65d10d346f4e -10.065,201,3.2,65d10d346f4e -10.178,201,3.1,65d10d346f4e -10.288,201,3.5,65d10d346f4e -10.401,201,6.0,c2f5d82dee7c -10.514,201,3.7,c2f5d82dee7c -10.626,201,4.8,c2f5d82dee7c -10.74,201,4.1,c2f5d82dee7c -10.853,201,4.6,65d10d346f4e -10.963,201,3.2,65d10d346f4e -11.072,201,4.3,65d10d346f4e -11.186,201,4.1,65d10d346f4e -11.291,201,7.7,65d10d346f4e -11.406,201,7.2,65d10d346f4e -11.52,201,4.9,65d10d346f4e -11.633,201,2.9,c2f5d82dee7c -11.745,201,3.3,c2f5d82dee7c -11.854,201,3.3,c2f5d82dee7c -11.968,201,3.0,c2f5d82dee7c -12.078,201,4.7,c2f5d82dee7c -12.191,201,4.8,65d10d346f4e -12.3,201,6.5,c2f5d82dee7c -12.416,201,4.8,65d10d346f4e -12.531,201,4.5,65d10d346f4e -12.646,201,4.0,c2f5d82dee7c -12.756,201,3.4,c2f5d82dee7c -12.86,201,3.0,65d10d346f4e -12.97,201,3.1,65d10d346f4e -13.08,201,4.9,c2f5d82dee7c -13.192,201,5.3,c2f5d82dee7c -13.306,201,3.7,c2f5d82dee7c -13.414,201,4.8,c2f5d82dee7c -13.526,201,5.3,c2f5d82dee7c -13.64,201,3.0,c2f5d82dee7c -13.751,201,3.6,c2f5d82dee7c -13.862,201,4.3,65d10d346f4e -13.976,201,4.5,65d10d346f4e -14.085,201,3.8,65d10d346f4e -14.193,201,4.3,65d10d346f4e -14.307,201,4.5,65d10d346f4e -14.415,201,3.9,65d10d346f4e -14.529,201,4.1,65d10d346f4e -14.635,201,3.1,65d10d346f4e -14.748,201,4.8,65d10d346f4e -14.863,201,5.1,c2f5d82dee7c -14.977,201,6.2,c2f5d82dee7c -15.089,201,3.8,c2f5d82dee7c -15.195,201,4.9,c2f5d82dee7c -15.306,201,3.7,c2f5d82dee7c -15.417,201,2.8,c2f5d82dee7c -15.523,201,3.8,c2f5d82dee7c -15.634,201,4.2,c2f5d82dee7c -15.738,201,5.9,65d10d346f4e -15.852,201,4.7,c2f5d82dee7c -15.966,201,5.5,c2f5d82dee7c -16.081,201,3.7,c2f5d82dee7c -16.186,201,3.3,c2f5d82dee7c -16.296,201,3.3,c2f5d82dee7c -16.405,201,2.9,c2f5d82dee7c -16.514,201,4.2,c2f5d82dee7c -16.621,201,4.5,65d10d346f4e -16.729,201,3.9,65d10d346f4e -16.838,201,4.1,65d10d346f4e -16.947,201,4.4,65d10d346f4e -17.054,201,4.1,65d10d346f4e -17.168,201,2.9,65d10d346f4e -17.271,201,4.7,65d10d346f4e -17.383,201,5.2,65d10d346f4e -17.495,201,5.8,65d10d346f4e -17.611,201,5.7,c2f5d82dee7c -17.722,201,4.9,c2f5d82dee7c -17.836,201,4.7,c2f5d82dee7c -17.947,201,5.4,c2f5d82dee7c -18.059,201,4.7,c2f5d82dee7c -18.17,201,5.1,c2f5d82dee7c -18.276,201,3.2,c2f5d82dee7c -18.387,201,3.3,c2f5d82dee7c -18.498,201,4.3,c2f5d82dee7c -18.603,201,4.5,c2f5d82dee7c -18.711,201,4.0,65d10d346f4e -18.819,201,4.4,65d10d346f4e -18.934,201,4.1,c2f5d82dee7c -19.041,201,4.7,65d10d346f4e -19.151,201,3.4,65d10d346f4e -19.262,201,3.6,65d10d346f4e -19.376,201,4.4,c2f5d82dee7c -19.48,201,4.4,65d10d346f4e -19.586,201,6.1,65d10d346f4e -19.702,201,5.3,c2f5d82dee7c -19.812,201,5.8,c2f5d82dee7c -19.922,201,4.5,65d10d346f4e -20.036,201,9.7,c2f5d82dee7c -20.155,201,11.4,65d10d346f4e -20.266,201,41.3,65d10d346f4e -20.415,201,7.7,65d10d346f4e -20.526,201,11.3,65d10d346f4e -20.645,201,126.5,65d10d346f4e -20.881,201,21.0,65d10d346f4e -21.009,201,29.7,c2f5d82dee7c -21.142,201,4.7,65d10d346f4e -21.246,201,3.4,c2f5d82dee7c -21.356,201,4.2,c2f5d82dee7c -21.466,201,3.7,c2f5d82dee7c -21.57,201,4.9,65d10d346f4e -21.676,201,3.5,65d10d346f4e -21.787,201,3.9,65d10d346f4e -21.893,201,3.9,c2f5d82dee7c -22.005,201,4.1,65d10d346f4e -22.111,201,5.0,65d10d346f4e -22.217,201,3.8,c2f5d82dee7c -22.331,201,4.5,65d10d346f4e -22.439,201,4.6,c2f5d82dee7c -22.553,201,2.9,65d10d346f4e -22.666,201,2.8,65d10d346f4e -22.777,201,4.8,65d10d346f4e -22.884,201,7.5,65d10d346f4e -22.994,201,5.6,65d10d346f4e -23.109,201,5.1,65d10d346f4e -23.223,201,5.4,c2f5d82dee7c -23.338,201,4.8,c2f5d82dee7c -23.453,201,5.0,65d10d346f4e -23.563,201,3.6,65d10d346f4e -23.67,201,4.6,c2f5d82dee7c -23.782,201,3.9,65d10d346f4e -23.887,201,4.4,c2f5d82dee7c -24.001,201,4.4,65d10d346f4e -24.105,201,3.9,c2f5d82dee7c -24.215,201,4.1,c2f5d82dee7c -24.321,201,2.7,65d10d346f4e -24.433,201,2.7,c2f5d82dee7c -24.537,201,2.8,65d10d346f4e -24.649,201,3.6,65d10d346f4e -24.757,201,3.9,65d10d346f4e -24.867,201,3.4,65d10d346f4e -24.977,201,2.9,65d10d346f4e -25.086,201,3.7,65d10d346f4e -25.195,201,3.3,65d10d346f4e -25.301,201,2.8,c2f5d82dee7c -25.412,201,4.2,65d10d346f4e -25.521,201,3.1,c2f5d82dee7c -25.626,201,4.5,c2f5d82dee7c -25.739,201,4.9,c2f5d82dee7c -25.844,201,3.3,c2f5d82dee7c -25.957,201,3.6,c2f5d82dee7c -26.061,201,3.4,65d10d346f4e -26.166,201,3.4,65d10d346f4e -26.275,201,2.7,65d10d346f4e -26.385,201,4.5,65d10d346f4e -26.492,201,4.3,c2f5d82dee7c -26.601,201,4.0,65d10d346f4e -26.708,201,3.9,c2f5d82dee7c -26.817,201,3.8,c2f5d82dee7c -26.925,201,5.6,c2f5d82dee7c -27.034,201,4.1,c2f5d82dee7c -27.141,201,3.2,65d10d346f4e -27.251,201,3.1,65d10d346f4e -27.357,201,4.3,65d10d346f4e -27.467,201,2.8,65d10d346f4e -27.576,201,3.3,65d10d346f4e -27.682,201,4.0,c2f5d82dee7c -27.788,201,3.2,c2f5d82dee7c -27.9,201,3.9,c2f5d82dee7c -28.011,201,5.4,c2f5d82dee7c -28.123,201,4.6,65d10d346f4e -28.229,201,4.5,c2f5d82dee7c -28.335,201,3.7,c2f5d82dee7c -28.449,201,3.8,c2f5d82dee7c -28.554,201,4.5,c2f5d82dee7c -28.666,201,3.9,c2f5d82dee7c -28.779,201,3.4,c2f5d82dee7c -28.884,201,2.6,c2f5d82dee7c -28.994,201,4.8,c2f5d82dee7c -29.103,201,3.8,65d10d346f4e -29.214,201,5.0,c2f5d82dee7c -29.324,201,6.4,c2f5d82dee7c -29.438,201,4.1,c2f5d82dee7c -29.552,201,2.6,c2f5d82dee7c -29.664,201,2.5,65d10d346f4e -29.767,201,3.2,c2f5d82dee7c -29.88,201,14.2,c2f5d82dee7c -30.002,201,4.0,c2f5d82dee7c -30.108,201,4.3,65d10d346f4e -30.216,201,4.1,65d10d346f4e -30.328,201,4.2,65d10d346f4e -30.439,201,3.8,65d10d346f4e -30.551,201,2.7,65d10d346f4e -30.661,201,2.6,65d10d346f4e -30.773,201,4.8,c2f5d82dee7c -30.88,201,3.7,65d10d346f4e -30.994,201,3.9,c2f5d82dee7c -31.101,201,3.8,65d10d346f4e -31.208,201,3.9,65d10d346f4e -31.32,201,3.4,65d10d346f4e -31.431,201,2.9,65d10d346f4e -31.538,201,2.9,65d10d346f4e -31.644,201,4.0,65d10d346f4e -31.749,201,5.8,c2f5d82dee7c -31.864,201,4.7,65d10d346f4e -31.979,201,3.9,65d10d346f4e -32.087,201,3.8,65d10d346f4e -32.196,201,3.0,65d10d346f4e -32.304,201,4.2,65d10d346f4e -32.415,201,3.3,65d10d346f4e -32.527,201,4.9,65d10d346f4e -32.634,201,3.6,65d10d346f4e -32.741,201,6.3,65d10d346f4e -32.857,201,5.0,65d10d346f4e -32.965,201,3.0,65d10d346f4e -33.073,201,4.4,65d10d346f4e -33.181,201,3.3,c2f5d82dee7c -33.289,201,2.9,c2f5d82dee7c -33.399,201,4.7,c2f5d82dee7c -33.513,201,4.1,65d10d346f4e -33.622,201,5.1,65d10d346f4e -33.737,201,4.9,65d10d346f4e -33.848,201,3.0,65d10d346f4e -33.957,201,4.5,65d10d346f4e -34.068,201,3.9,65d10d346f4e -34.182,201,3.0,c2f5d82dee7c -34.292,201,2.8,65d10d346f4e -34.403,201,3.9,c2f5d82dee7c -34.515,201,2.9,c2f5d82dee7c -34.62,201,3.3,65d10d346f4e -34.725,201,4.4,65d10d346f4e -34.834,201,2.7,65d10d346f4e -34.946,201,3.9,65d10d346f4e -35.058,201,2.8,65d10d346f4e -35.161,201,3.0,c2f5d82dee7c -35.27,201,3.4,c2f5d82dee7c -35.376,201,4.1,c2f5d82dee7c -35.487,201,3.1,c2f5d82dee7c -35.6,201,3.2,c2f5d82dee7c -35.708,201,2.9,c2f5d82dee7c -35.821,201,3.3,c2f5d82dee7c -35.931,201,4.5,65d10d346f4e -36.045,201,5.2,65d10d346f4e -36.154,201,4.3,65d10d346f4e -36.261,201,4.1,c2f5d82dee7c -36.374,201,4.6,65d10d346f4e -36.489,201,5.7,65d10d346f4e -36.605,201,3.4,c2f5d82dee7c -36.714,201,3.2,c2f5d82dee7c -36.827,201,4.6,c2f5d82dee7c -36.938,201,5.1,c2f5d82dee7c -37.053,201,4.7,65d10d346f4e -37.158,201,4.4,c2f5d82dee7c -37.268,201,3.5,c2f5d82dee7c -37.382,201,4.3,65d10d346f4e -37.496,201,3.7,65d10d346f4e -37.6,201,3.4,c2f5d82dee7c -37.71,201,3.5,c2f5d82dee7c -37.817,201,5.5,c2f5d82dee7c -37.93,201,5.0,c2f5d82dee7c -38.044,201,6.7,65d10d346f4e -38.151,201,6.6,65d10d346f4e -38.259,201,4.2,c2f5d82dee7c -38.37,201,4.1,c2f5d82dee7c -38.475,201,3.6,c2f5d82dee7c -38.585,201,3.2,c2f5d82dee7c -38.694,201,4.9,c2f5d82dee7c -38.808,201,4.2,c2f5d82dee7c -38.915,201,3.2,c2f5d82dee7c -39.024,201,5.7,c2f5d82dee7c -39.135,201,4.5,c2f5d82dee7c -39.24,201,4.9,65d10d346f4e -39.347,201,4.8,65d10d346f4e -39.453,201,3.6,65d10d346f4e -39.559,201,4.4,c2f5d82dee7c -39.674,201,4.0,65d10d346f4e -39.78,201,4.7,c2f5d82dee7c -39.888,201,4.2,c2f5d82dee7c -39.995,201,5.7,c2f5d82dee7c -40.107,201,3.2,c2f5d82dee7c -40.21,201,4.6,65d10d346f4e -40.316,201,4.9,65d10d346f4e -40.431,201,3.6,65d10d346f4e -40.537,201,3.4,65d10d346f4e -40.65,201,3.8,65d10d346f4e -40.763,201,4.7,c2f5d82dee7c -40.873,201,4.0,c2f5d82dee7c -40.979,201,3.9,65d10d346f4e -41.087,201,3.6,65d10d346f4e -41.201,201,4.6,65d10d346f4e -41.313,201,3.0,65d10d346f4e -41.416,201,3.7,65d10d346f4e -41.524,201,3.9,65d10d346f4e -41.63,201,4.7,c2f5d82dee7c -41.74,201,3.9,c2f5d82dee7c -41.851,201,3.9,c2f5d82dee7c -41.962,201,4.2,65d10d346f4e -42.072,201,2.8,c2f5d82dee7c -42.184,201,2.8,65d10d346f4e -42.297,201,2.7,65d10d346f4e -42.41,201,3.3,65d10d346f4e -42.519,201,4.3,65d10d346f4e -42.626,201,3.9,65d10d346f4e -42.734,201,3.8,65d10d346f4e -42.84,201,4.2,c2f5d82dee7c -42.95,201,3.3,c2f5d82dee7c -43.057,201,3.5,c2f5d82dee7c -43.165,201,3.4,c2f5d82dee7c -43.278,201,3.6,c2f5d82dee7c -43.382,201,5.9,65d10d346f4e -43.498,201,4.6,c2f5d82dee7c -43.605,201,4.0,c2f5d82dee7c -43.714,201,4.1,c2f5d82dee7c -43.826,201,4.0,c2f5d82dee7c -43.936,201,2.6,c2f5d82dee7c -44.046,201,3.0,c2f5d82dee7c -44.155,201,3.9,c2f5d82dee7c -44.267,201,4.2,c2f5d82dee7c -44.375,201,3.9,c2f5d82dee7c -44.482,201,9.0,c2f5d82dee7c -44.598,201,4.3,c2f5d82dee7c -44.706,201,3.1,c2f5d82dee7c -44.813,201,2.6,c2f5d82dee7c -44.922,201,2.8,c2f5d82dee7c -45.03,201,3.9,65d10d346f4e -45.144,201,3.7,c2f5d82dee7c -45.255,201,3.5,c2f5d82dee7c -45.368,201,4.0,c2f5d82dee7c -45.475,201,3.6,c2f5d82dee7c -45.584,201,3.6,c2f5d82dee7c -45.693,201,3.1,65d10d346f4e -45.803,201,7.3,c2f5d82dee7c -45.911,201,2.7,65d10d346f4e -46.023,201,4.4,c2f5d82dee7c -46.137,201,4.1,c2f5d82dee7c -46.244,201,3.4,c2f5d82dee7c -46.349,201,4.8,65d10d346f4e -46.462,201,3.0,c2f5d82dee7c -46.575,201,3.6,c2f5d82dee7c -46.682,201,3.8,c2f5d82dee7c -46.793,201,4.3,c2f5d82dee7c -46.905,201,3.5,c2f5d82dee7c -47.017,201,3.9,c2f5d82dee7c -47.13,201,5.4,65d10d346f4e -47.24,201,3.3,c2f5d82dee7c -47.349,201,3.4,c2f5d82dee7c -47.452,201,2.8,c2f5d82dee7c -47.559,201,2.6,65d10d346f4e -47.67,201,3.0,65d10d346f4e -47.775,201,3.1,65d10d346f4e -47.878,201,6.3,c2f5d82dee7c -47.989,201,5.9,c2f5d82dee7c -48.105,201,5.2,65d10d346f4e -48.219,201,4.1,65d10d346f4e -48.333,201,4.8,c2f5d82dee7c -48.437,201,5.5,65d10d346f4e -48.543,201,4.8,c2f5d82dee7c -48.65,201,4.7,c2f5d82dee7c -48.764,201,4.3,65d10d346f4e -48.878,201,3.6,65d10d346f4e -48.983,201,3.8,65d10d346f4e -49.087,201,4.9,c2f5d82dee7c -49.201,201,5.1,c2f5d82dee7c -49.314,201,3.0,65d10d346f4e -49.421,201,3.4,c2f5d82dee7c -49.527,201,3.4,c2f5d82dee7c -49.633,201,5.0,c2f5d82dee7c -49.743,201,4.7,c2f5d82dee7c -49.853,201,4.8,c2f5d82dee7c -49.968,201,4.3,c2f5d82dee7c -50.082,201,3.4,c2f5d82dee7c -50.193,201,2.8,65d10d346f4e -50.298,201,2.8,c2f5d82dee7c -50.407,201,5.2,c2f5d82dee7c -50.514,201,4.1,c2f5d82dee7c -50.621,201,5.9,c2f5d82dee7c -50.729,201,3.7,65d10d346f4e -50.838,201,3.0,65d10d346f4e -50.951,201,3.2,65d10d346f4e -51.06,201,2.8,65d10d346f4e -51.169,201,3.4,65d10d346f4e -51.282,201,5.1,c2f5d82dee7c -51.397,201,4.1,c2f5d82dee7c -51.506,201,4.2,c2f5d82dee7c -51.62,201,4.0,c2f5d82dee7c -51.726,201,3.8,c2f5d82dee7c -51.84,201,3.9,c2f5d82dee7c -51.949,201,4.4,c2f5d82dee7c -52.055,201,3.0,c2f5d82dee7c -52.159,201,4.9,65d10d346f4e -52.273,201,3.8,c2f5d82dee7c -52.386,201,5.2,c2f5d82dee7c -52.499,201,3.7,c2f5d82dee7c -52.602,201,4.2,c2f5d82dee7c -52.716,201,3.5,c2f5d82dee7c -52.827,201,3.4,c2f5d82dee7c -52.936,201,4.1,c2f5d82dee7c -53.047,201,6.5,c2f5d82dee7c -53.159,201,6.6,c2f5d82dee7c -53.272,201,7.8,65d10d346f4e -53.388,201,4.3,65d10d346f4e -53.496,201,5.3,65d10d346f4e -53.605,201,4.0,65d10d346f4e -53.71,201,4.0,c2f5d82dee7c -53.823,201,4.8,65d10d346f4e -53.937,201,4.6,65d10d346f4e -54.05,201,4.2,65d10d346f4e -54.164,201,4.4,c2f5d82dee7c -54.272,201,4.6,c2f5d82dee7c -54.383,201,4.6,c2f5d82dee7c -54.497,201,4.0,c2f5d82dee7c -54.611,201,4.8,65d10d346f4e -54.722,201,3.9,65d10d346f4e -54.827,201,4.9,c2f5d82dee7c -54.937,201,4.1,c2f5d82dee7c -55.046,201,4.6,c2f5d82dee7c -55.153,201,4.6,c2f5d82dee7c -55.265,201,3.7,c2f5d82dee7c -55.378,201,3.4,c2f5d82dee7c -55.483,201,3.3,c2f5d82dee7c -55.587,201,3.0,65d10d346f4e -55.699,201,4.1,65d10d346f4e -55.813,201,4.8,c2f5d82dee7c -55.927,201,4.4,c2f5d82dee7c -56.041,201,4.9,65d10d346f4e -56.156,201,3.8,65d10d346f4e -56.267,201,5.6,65d10d346f4e -56.375,201,4.1,65d10d346f4e -56.481,201,3.2,c2f5d82dee7c -56.588,201,3.3,c2f5d82dee7c -56.699,201,6.7,c2f5d82dee7c -56.815,201,4.7,65d10d346f4e -56.926,201,4.6,65d10d346f4e -57.04,201,4.4,65d10d346f4e -57.144,201,4.6,65d10d346f4e -57.257,201,3.7,65d10d346f4e -57.362,201,3.0,c2f5d82dee7c -57.475,201,3.8,65d10d346f4e -57.589,201,4.3,65d10d346f4e -57.699,201,4.2,65d10d346f4e -57.811,201,5.7,c2f5d82dee7c -57.919,201,5.7,65d10d346f4e -58.033,201,6.8,c2f5d82dee7c -58.144,201,8.0,c2f5d82dee7c -58.252,201,4.6,65d10d346f4e -58.359,201,6.6,65d10d346f4e -58.475,201,4.5,c2f5d82dee7c -58.585,201,4.6,c2f5d82dee7c -58.699,201,5.3,c2f5d82dee7c -58.813,201,5.6,65d10d346f4e -58.922,201,3.5,c2f5d82dee7c -59.036,201,3.3,65d10d346f4e -59.147,201,3.0,65d10d346f4e -59.255,201,3.2,65d10d346f4e -59.362,201,3.3,65d10d346f4e -59.47,201,3.3,c2f5d82dee7c -59.577,201,5.1,c2f5d82dee7c -59.689,201,3.3,c2f5d82dee7c -59.797,201,3.5,c2f5d82dee7c -59.908,201,2.5,c2f5d82dee7c -60.015,201,3.9,c2f5d82dee7c -60.126,201,3.8,c2f5d82dee7c -60.233,201,3.5,c2f5d82dee7c -60.343,201,3.6,c2f5d82dee7c -60.447,201,4.0,65d10d346f4e -60.555,201,4.2,65d10d346f4e -60.668,201,3.9,65d10d346f4e -60.778,201,2.8,65d10d346f4e -60.886,201,2.6,65d10d346f4e -60.997,201,4.1,65d10d346f4e -61.112,201,4.0,c2f5d82dee7c -61.225,201,4.8,c2f5d82dee7c -61.335,201,4.1,c2f5d82dee7c -61.449,201,4.2,c2f5d82dee7c -61.559,201,4.0,c2f5d82dee7c -61.666,201,2.8,c2f5d82dee7c -61.774,201,4.9,c2f5d82dee7c -61.887,201,4.6,c2f5d82dee7c -62.002,201,4.2,65d10d346f4e -62.116,201,3.8,65d10d346f4e -62.224,201,4.5,65d10d346f4e -62.336,201,4.8,65d10d346f4e -62.45,201,3.2,65d10d346f4e -62.559,201,4.3,65d10d346f4e -62.674,201,4.7,c2f5d82dee7c -62.779,201,6.9,65d10d346f4e -62.889,201,5.8,65d10d346f4e -62.997,201,5.6,65d10d346f4e -63.111,201,4.6,c2f5d82dee7c -63.226,201,4.5,c2f5d82dee7c -63.341,201,4.6,65d10d346f4e -63.454,201,3.9,65d10d346f4e -63.56,201,4.3,c2f5d82dee7c -63.669,201,5.7,c2f5d82dee7c -63.78,201,6.2,c2f5d82dee7c -63.894,201,4.3,65d10d346f4e -64.001,201,4.0,65d10d346f4e -64.115,201,8.0,65d10d346f4e -64.233,201,3.7,c2f5d82dee7c -64.341,201,3.9,65d10d346f4e -64.454,201,3.6,c2f5d82dee7c -64.565,201,3.7,c2f5d82dee7c -64.673,201,4.2,c2f5d82dee7c -64.781,201,4.3,c2f5d82dee7c -64.888,201,4.0,65d10d346f4e -64.997,201,3.8,65d10d346f4e -65.104,201,2.8,65d10d346f4e -65.209,201,2.8,c2f5d82dee7c -65.318,201,2.5,c2f5d82dee7c -65.428,201,4.1,c2f5d82dee7c -65.534,201,3.2,c2f5d82dee7c -65.642,201,5.3,c2f5d82dee7c -65.757,201,4.7,c2f5d82dee7c -65.864,201,5.2,c2f5d82dee7c -65.978,201,3.0,c2f5d82dee7c -66.089,201,2.8,c2f5d82dee7c -66.196,201,3.1,c2f5d82dee7c -66.301,201,3.4,65d10d346f4e -66.411,201,3.7,65d10d346f4e -66.521,201,5.3,c2f5d82dee7c -66.635,201,4.7,c2f5d82dee7c -66.744,201,4.1,c2f5d82dee7c -66.849,201,3.0,65d10d346f4e -66.954,201,3.2,65d10d346f4e -67.066,201,3.6,65d10d346f4e -67.172,201,5.1,65d10d346f4e -67.281,201,3.7,c2f5d82dee7c -67.39,201,6.3,65d10d346f4e -67.498,201,6.4,c2f5d82dee7c -67.607,201,4.7,c2f5d82dee7c -67.715,201,6.8,c2f5d82dee7c -67.829,201,3.7,c2f5d82dee7c -67.934,201,4.2,c2f5d82dee7c -68.046,201,4.4,c2f5d82dee7c -68.16,201,2.7,c2f5d82dee7c -68.268,201,5.6,c2f5d82dee7c -68.382,201,4.0,65d10d346f4e -68.489,201,4.2,c2f5d82dee7c -68.598,201,3.8,c2f5d82dee7c -68.706,201,5.8,c2f5d82dee7c -68.819,201,2.9,c2f5d82dee7c -68.931,201,3.2,c2f5d82dee7c -69.044,201,3.2,65d10d346f4e -69.158,201,4.4,65d10d346f4e -69.264,201,5.1,65d10d346f4e -69.376,201,4.1,65d10d346f4e -69.485,201,4.5,65d10d346f4e -69.594,201,4.2,65d10d346f4e -69.704,201,4.5,65d10d346f4e -69.816,201,3.3,65d10d346f4e -69.925,201,2.9,65d10d346f4e -70.035,201,5.1,65d10d346f4e -70.145,201,3.5,65d10d346f4e -70.256,201,3.6,65d10d346f4e -70.37,201,4.0,65d10d346f4e -70.484,201,4.2,c2f5d82dee7c -70.598,201,5.0,c2f5d82dee7c -70.713,201,4.5,65d10d346f4e -70.821,201,2.7,c2f5d82dee7c -70.933,201,4.5,65d10d346f4e -71.042,201,4.1,65d10d346f4e -71.149,201,3.4,c2f5d82dee7c -71.257,201,3.7,c2f5d82dee7c -71.362,201,3.5,c2f5d82dee7c -71.471,201,4.4,65d10d346f4e -71.585,201,2.8,c2f5d82dee7c -71.69,201,2.9,65d10d346f4e -71.803,201,5.1,c2f5d82dee7c -71.917,201,4.4,c2f5d82dee7c -72.027,201,3.9,c2f5d82dee7c -72.136,201,3.7,c2f5d82dee7c -72.248,201,4.8,c2f5d82dee7c -72.362,201,5.1,65d10d346f4e -72.475,201,3.7,65d10d346f4e -72.589,201,4.7,65d10d346f4e -72.697,201,5.0,65d10d346f4e -72.805,201,5.6,65d10d346f4e -72.911,201,5.4,c2f5d82dee7c -73.019,201,4.1,c2f5d82dee7c -73.129,201,5.2,c2f5d82dee7c -73.245,201,4.2,65d10d346f4e -73.355,201,3.2,65d10d346f4e -73.463,201,3.4,65d10d346f4e -73.572,201,3.5,65d10d346f4e -73.681,201,5.3,65d10d346f4e -73.787,201,3.6,c2f5d82dee7c -73.9,201,4.0,c2f5d82dee7c -74.008,201,4.4,c2f5d82dee7c -74.117,201,4.5,c2f5d82dee7c -74.225,201,9.1,c2f5d82dee7c -74.334,201,6.4,c2f5d82dee7c -74.448,201,3.5,c2f5d82dee7c -74.558,201,4.1,c2f5d82dee7c -74.672,201,3.9,65d10d346f4e -74.778,201,5.0,c2f5d82dee7c -74.886,201,3.8,c2f5d82dee7c -74.999,201,4.1,c2f5d82dee7c -75.113,201,5.5,65d10d346f4e -75.226,201,4.4,c2f5d82dee7c -75.337,201,4.2,65d10d346f4e -75.441,201,5.9,c2f5d82dee7c +-25.905,201,4.7,5bbf259ff136 +-25.8,201,7.2,7cfd91ddf780 +-25.693,201,8.3,5bbf259ff136 +-25.583,201,6.1,5bbf259ff136 +-25.474,201,4.9,7cfd91ddf780 +-25.363,201,6.4,7cfd91ddf780 +-25.249,201,4.4,5bbf259ff136 +-25.138,201,4.3,5bbf259ff136 +-25.027,201,4.9,5bbf259ff136 +-24.916,201,3.5,5bbf259ff136 +-24.807,201,3.6,5bbf259ff136 +-24.699,201,3.7,5bbf259ff136 +-24.595,201,3.7,7cfd91ddf780 +-24.484,201,4.4,7cfd91ddf780 +-24.371,201,6.0,5bbf259ff136 +-24.257,201,5.0,5bbf259ff136 +-24.149,201,5.0,5bbf259ff136 +-24.043,201,5.0,5bbf259ff136 +-23.932,201,6.6,5bbf259ff136 +-23.815,201,6.5,5bbf259ff136 +-23.699,201,6.6,7cfd91ddf780 +-23.592,201,5.2,7cfd91ddf780 +-23.484,201,6.0,7cfd91ddf780 +-23.374,201,5.3,5bbf259ff136 +-23.264,201,5.2,5bbf259ff136 +-23.151,201,5.0,7cfd91ddf780 +-23.038,201,5.2,7cfd91ddf780 +-22.923,201,6.2,5bbf259ff136 +-22.806,201,5.4,5bbf259ff136 +-22.691,201,6.0,7cfd91ddf780 +-22.575,201,5.8,7cfd91ddf780 +-22.459,201,5.1,5bbf259ff136 +-22.348,201,5.9,5bbf259ff136 +-22.232,201,6.2,7cfd91ddf780 +-22.122,201,5.8,7cfd91ddf780 +-22.007,201,6.7,7cfd91ddf780 +-21.897,201,5.1,7cfd91ddf780 +-21.789,201,5.1,7cfd91ddf780 +-21.674,201,6.1,7cfd91ddf780 +-21.558,201,5.6,5bbf259ff136 +-21.449,201,5.3,5bbf259ff136 +-21.333,201,4.5,7cfd91ddf780 +-21.225,201,4.9,5bbf259ff136 +-21.115,201,4.9,5bbf259ff136 +-21.0,201,3.6,7cfd91ddf780 +-20.889,201,4.4,7cfd91ddf780 +-20.778,201,5.1,7cfd91ddf780 +-20.665,201,4.1,7cfd91ddf780 +-20.556,201,4.3,7cfd91ddf780 +-20.444,201,3.3,7cfd91ddf780 +-20.34,201,3.5,7cfd91ddf780 +-20.231,201,4.4,7cfd91ddf780 +-20.124,201,6.0,7cfd91ddf780 +-20.017,201,5.2,5bbf259ff136 +-19.903,201,4.7,7cfd91ddf780 +-19.794,201,5.3,5bbf259ff136 +-19.683,201,5.4,7cfd91ddf780 +-19.574,201,5.3,5bbf259ff136 +-19.463,201,5.3,7cfd91ddf780 +-19.347,201,6.6,7cfd91ddf780 +-19.233,201,5.1,5bbf259ff136 +-19.124,201,4.6,7cfd91ddf780 +-19.009,201,4.9,5bbf259ff136 +-18.896,201,5.4,5bbf259ff136 +-18.781,201,5.4,7cfd91ddf780 +-18.667,201,6.9,7cfd91ddf780 +-18.55,201,5.1,5bbf259ff136 +-18.444,201,5.7,5bbf259ff136 +-18.336,201,4.8,7cfd91ddf780 +-18.225,201,6.3,7cfd91ddf780 +-18.118,201,5.2,7cfd91ddf780 +-18.002,201,6.5,5bbf259ff136 +-17.893,201,5.4,5bbf259ff136 +-17.777,201,6.7,5bbf259ff136 +-17.669,201,5.7,5bbf259ff136 +-17.558,201,6.1,5bbf259ff136 +-17.447,201,5.8,5bbf259ff136 +-17.336,201,5.3,5bbf259ff136 +-17.225,201,5.3,5bbf259ff136 +-17.112,201,6.0,7cfd91ddf780 +-16.996,201,5.3,7cfd91ddf780 +-16.885,201,5.1,7cfd91ddf780 +-16.771,201,5.6,5bbf259ff136 +-16.659,201,6.5,5bbf259ff136 +-16.544,201,5.7,5bbf259ff136 +-16.438,201,4.7,5bbf259ff136 +-16.332,201,5.4,5bbf259ff136 +-16.218,201,5.7,5bbf259ff136 +-16.112,201,5.5,5bbf259ff136 +-15.996,201,4.4,5bbf259ff136 +-15.887,201,3.2,5bbf259ff136 +-15.774,201,4.6,5bbf259ff136 +-15.66,201,6.1,7cfd91ddf780 +-15.544,201,6.2,5bbf259ff136 +-15.434,201,5.0,7cfd91ddf780 +-15.322,201,4.5,5bbf259ff136 +-15.215,201,4.1,7cfd91ddf780 +-15.104,201,6.0,5bbf259ff136 +-14.988,201,3.7,7cfd91ddf780 +-14.881,201,4.0,7cfd91ddf780 +-14.776,201,5.2,5bbf259ff136 +-14.663,201,4.9,5bbf259ff136 +-14.558,201,5.3,5bbf259ff136 +-14.45,201,4.9,5bbf259ff136 +-14.34,201,5.3,5bbf259ff136 +-14.233,201,5.4,5bbf259ff136 +-14.117,201,5.0,5bbf259ff136 +-14.011,201,4.9,5bbf259ff136 +-13.896,201,6.1,5bbf259ff136 +-13.781,201,7.1,7cfd91ddf780 +-13.672,201,6.6,7cfd91ddf780 +-13.565,201,5.0,5bbf259ff136 +-13.458,201,5.7,5bbf259ff136 +-13.342,201,5.3,7cfd91ddf780 +-13.236,201,5.6,5bbf259ff136 +-13.124,201,7.3,7cfd91ddf780 +-13.007,201,5.8,7cfd91ddf780 +-12.892,201,8.0,5bbf259ff136 +-12.776,201,5.1,5bbf259ff136 +-12.661,201,5.0,7cfd91ddf780 +-12.55,201,4.5,7cfd91ddf780 +-12.441,201,5.3,7cfd91ddf780 +-12.327,201,5.8,7cfd91ddf780 +-12.219,201,6.0,7cfd91ddf780 +-12.106,201,4.7,7cfd91ddf780 +-11.999,201,6.4,7cfd91ddf780 +-11.888,201,4.7,7cfd91ddf780 +-11.777,201,4.8,7cfd91ddf780 +-11.672,201,5.3,7cfd91ddf780 +-11.557,201,4.5,7cfd91ddf780 +-11.444,201,4.8,7cfd91ddf780 +-11.33,201,5.7,5bbf259ff136 +-11.222,201,6.4,5bbf259ff136 +-11.108,201,5.1,5bbf259ff136 +-11.0,201,4.3,5bbf259ff136 +-10.893,201,4.8,7cfd91ddf780 +-10.782,201,4.2,5bbf259ff136 +-10.671,201,4.6,5bbf259ff136 +-10.558,201,6.0,5bbf259ff136 +-10.449,201,5.6,5bbf259ff136 +-10.335,201,5.1,5bbf259ff136 +-10.23,201,4.8,5bbf259ff136 +-10.115,201,3.9,5bbf259ff136 +-10.009,201,3.9,5bbf259ff136 +-9.901,201,5.4,5bbf259ff136 +-9.785,201,4.9,5bbf259ff136 +-9.673,201,5.6,7cfd91ddf780 +-9.566,201,5.0,5bbf259ff136 +-9.458,201,6.3,5bbf259ff136 +-9.35,201,5.3,5bbf259ff136 +-9.235,201,6.5,5bbf259ff136 +-9.125,201,5.8,5bbf259ff136 +-9.009,201,4.8,7cfd91ddf780 +-8.895,201,6.3,7cfd91ddf780 +-8.783,201,5.3,5bbf259ff136 +-8.674,201,5.7,7cfd91ddf780 +-8.566,201,6.7,7cfd91ddf780 +-8.455,201,3.6,7cfd91ddf780 +-8.349,201,5.7,7cfd91ddf780 +-8.239,201,4.5,7cfd91ddf780 +-8.126,201,5.6,7cfd91ddf780 +-8.015,201,5.7,7cfd91ddf780 +-7.903,201,6.2,7cfd91ddf780 +-7.796,201,5.0,7cfd91ddf780 +-7.682,201,5.6,5bbf259ff136 +-7.57,201,5.2,5bbf259ff136 +-7.456,201,6.0,5bbf259ff136 +-7.35,201,5.2,5bbf259ff136 +-7.234,201,5.1,5bbf259ff136 +-7.122,201,4.7,7cfd91ddf780 +-7.014,201,6.2,5bbf259ff136 +-6.902,201,4.9,7cfd91ddf780 +-6.787,201,4.8,7cfd91ddf780 +-6.674,201,5.6,7cfd91ddf780 +-6.566,201,5.3,7cfd91ddf780 +-6.452,201,6.5,5bbf259ff136 +-6.341,201,5.5,5bbf259ff136 +-6.226,201,6.0,5bbf259ff136 +-6.118,201,5.4,5bbf259ff136 +-6.002,201,5.6,7cfd91ddf780 +-5.888,201,5.6,7cfd91ddf780 +-5.773,201,5.1,7cfd91ddf780 +-5.658,201,3.6,5bbf259ff136 +-5.547,201,3.6,5bbf259ff136 +-5.442,201,5.2,5bbf259ff136 +-5.332,201,5.4,5bbf259ff136 +-5.216,201,5.2,5bbf259ff136 +-5.102,201,4.5,7cfd91ddf780 +-4.988,201,4.1,7cfd91ddf780 +-4.876,201,3.6,7cfd91ddf780 +-4.771,201,5.1,7cfd91ddf780 +-4.657,201,5.5,7cfd91ddf780 +-4.543,201,5.2,7cfd91ddf780 +-4.428,201,7.1,5bbf259ff136 +-4.314,201,5.3,5bbf259ff136 +-4.208,201,5.0,5bbf259ff136 +-4.1,201,6.9,5bbf259ff136 +-3.992,201,4.4,5bbf259ff136 +-3.884,201,5.1,7cfd91ddf780 +-3.773,201,5.9,5bbf259ff136 +-3.657,201,5.3,5bbf259ff136 +-3.542,201,5.6,7cfd91ddf780 +-3.431,201,5.8,7cfd91ddf780 +-3.32,201,4.9,7cfd91ddf780 +-3.208,201,5.1,7cfd91ddf780 +-3.095,201,5.1,7cfd91ddf780 +-2.988,201,7.0,7cfd91ddf780 +-2.874,201,6.1,7cfd91ddf780 +-2.768,201,6.5,7cfd91ddf780 +-2.652,201,5.2,5bbf259ff136 +-2.539,201,6.3,5bbf259ff136 +-2.423,201,7.4,7cfd91ddf780 +-2.315,201,6.9,7cfd91ddf780 +-2.201,201,6.5,7cfd91ddf780 +-2.086,201,5.1,7cfd91ddf780 +-1.975,201,5.3,7cfd91ddf780 +-1.859,201,6.2,5bbf259ff136 +-1.747,201,5.0,5bbf259ff136 +-1.642,201,5.4,5bbf259ff136 +-1.535,201,4.7,7cfd91ddf780 +-1.42,201,5.7,5bbf259ff136 +-1.304,201,5.2,5bbf259ff136 +-1.191,201,5.2,7cfd91ddf780 +-1.084,201,6.3,5bbf259ff136 +-0.972,201,5.8,7cfd91ddf780 +-0.86,201,5.1,7cfd91ddf780 +-0.745,201,6.2,7cfd91ddf780 +-0.632,201,5.1,5bbf259ff136 +-0.517,201,3.6,5bbf259ff136 +-0.403,201,4.9,7cfd91ddf780 +-0.288,201,5.7,7cfd91ddf780 +-0.178,201,4.9,7cfd91ddf780 +-0.065,201,7.0,7cfd91ddf780 +0.05,201,4.6,5bbf259ff136 +0.155,201,4.1,7cfd91ddf780 +0.259,201,4.2,7cfd91ddf780 +0.373,201,3.1,7cfd91ddf780 +0.481,201,3.9,7cfd91ddf780 +0.592,201,5.4,7cfd91ddf780 +0.699,201,5.0,7cfd91ddf780 +0.805,201,7.4,7cfd91ddf780 +0.922,201,6.3,7cfd91ddf780 +1.039,201,7.8,5bbf259ff136 +1.148,201,5.0,5bbf259ff136 +1.262,201,4.5,5bbf259ff136 +1.367,201,3.7,7cfd91ddf780 +1.471,201,3.2,7cfd91ddf780 +1.575,201,5.0,5bbf259ff136 +1.681,201,4.6,5bbf259ff136 +1.792,201,3.6,5bbf259ff136 +1.897,201,5.6,5bbf259ff136 +2.013,201,3.7,5bbf259ff136 +2.125,201,4.8,5bbf259ff136 +2.23,201,5.8,5bbf259ff136 +2.345,201,3.2,5bbf259ff136 +2.457,201,4.2,7cfd91ddf780 +2.569,201,2.9,7cfd91ddf780 +2.677,201,4.0,5bbf259ff136 +2.781,201,6.1,5bbf259ff136 +2.89,201,4.8,5bbf259ff136 +3.001,201,5.8,5bbf259ff136 +3.108,201,9.4,7cfd91ddf780 +3.226,201,4.3,7cfd91ddf780 +3.336,201,3.9,7cfd91ddf780 +3.45,201,3.6,a352bb0a17ff +3.562,201,3.4,a352bb0a17ff +3.666,201,2.9,7cfd91ddf780 +3.769,201,3.8,7cfd91ddf780 +3.877,201,4.6,5bbf259ff136 +3.989,201,3.1,7cfd91ddf780 +4.099,201,3.7,7cfd91ddf780 +4.203,201,4.1,5bbf259ff136 +4.309,201,3.1,5bbf259ff136 +4.415,201,5.6,a352bb0a17ff +4.522,201,6.2,a352bb0a17ff +4.632,201,3.4,a352bb0a17ff +4.743,201,8.9,a352bb0a17ff +4.858,201,3.0,5bbf259ff136 +4.966,201,3.6,a352bb0a17ff +5.071,201,4.0,a352bb0a17ff +5.182,201,3.9,a352bb0a17ff +5.294,201,3.0,a352bb0a17ff +5.407,201,3.6,5bbf259ff136 +5.517,201,5.7,a352bb0a17ff +5.628,201,6.4,5bbf259ff136 +5.738,201,4.2,5bbf259ff136 +5.847,201,5.3,5bbf259ff136 +5.956,201,4.5,a352bb0a17ff +6.066,201,4.1,a352bb0a17ff +6.175,201,3.8,a352bb0a17ff +6.289,201,4.2,5bbf259ff136 +6.4,201,3.8,5bbf259ff136 +6.508,201,3.9,5bbf259ff136 +6.615,201,3.9,a352bb0a17ff +6.728,201,4.2,5bbf259ff136 +6.841,201,4.4,5bbf259ff136 +6.956,201,3.5,5bbf259ff136 +7.066,201,4.1,5bbf259ff136 +7.176,201,5.2,a352bb0a17ff +7.286,201,4.1,5bbf259ff136 +7.398,201,3.7,a352bb0a17ff +7.51,201,5.2,a352bb0a17ff +7.624,201,4.6,a352bb0a17ff +7.733,201,4.2,a352bb0a17ff +7.846,201,4.1,a352bb0a17ff +7.958,201,4.1,f6a858d7a20b +8.069,201,3.9,f6a858d7a20b +8.177,201,5.1,f6a858d7a20b +8.29,201,5.5,f6a858d7a20b +8.405,201,6.0,f6a858d7a20b +8.517,201,5.5,5bbf259ff136 +8.626,201,5.2,f6a858d7a20b +8.736,201,6.0,a352bb0a17ff +8.847,201,5.7,f6a858d7a20b +8.963,201,5.3,f6a858d7a20b +9.078,201,7.2,a352bb0a17ff +9.186,201,6.3,a352bb0a17ff +9.3,201,6.4,a352bb0a17ff +9.41,201,4.9,a352bb0a17ff +9.52,201,5.1,a352bb0a17ff +9.632,201,4.6,a352bb0a17ff +9.747,201,5.6,f6a858d7a20b +9.862,201,3.8,f6a858d7a20b +9.976,201,5.5,a352bb0a17ff +10.092,201,6.9,a352bb0a17ff +10.203,201,4.5,a352bb0a17ff +10.309,201,5.3,a352bb0a17ff +10.425,201,5.3,f6a858d7a20b +10.54,201,3.4,f6a858d7a20b +10.643,201,4.0,a352bb0a17ff +10.748,201,3.7,a352bb0a17ff +10.854,201,3.7,f6a858d7a20b +10.959,201,6.0,f6a858d7a20b +11.075,201,4.4,f6a858d7a20b +11.189,201,5.0,a352bb0a17ff +11.302,201,3.4,a352bb0a17ff +11.41,201,3.9,a352bb0a17ff +11.514,201,4.4,f6a858d7a20b +11.626,201,4.8,f6a858d7a20b +11.733,201,5.1,f6a858d7a20b +11.848,201,5.0,a352bb0a17ff +11.958,201,4.5,a352bb0a17ff +12.065,201,5.0,f6a858d7a20b +12.175,201,3.5,f6a858d7a20b +12.286,201,3.6,f6a858d7a20b +12.391,201,3.9,f6a858d7a20b +12.502,201,5.0,f6a858d7a20b +12.617,201,5.1,a352bb0a17ff +12.725,201,5.8,f6a858d7a20b +12.832,201,3.5,f6a858d7a20b +12.937,201,4.1,f6a858d7a20b +13.05,201,5.3,f6a858d7a20b +13.161,201,5.3,f6a858d7a20b +13.276,201,5.0,a352bb0a17ff +13.391,201,5.2,a352bb0a17ff +13.507,201,5.4,f6a858d7a20b +13.614,201,7.0,a352bb0a17ff +13.729,201,5.0,f6a858d7a20b +13.844,201,4.6,f6a858d7a20b +13.957,201,7.2,a352bb0a17ff +14.072,201,5.4,a352bb0a17ff +14.183,201,5.6,a352bb0a17ff +14.298,201,5.2,f6a858d7a20b +14.411,201,5.5,f6a858d7a20b +14.522,201,4.7,f6a858d7a20b +14.637,201,5.5,a352bb0a17ff +14.753,201,5.2,a352bb0a17ff +14.864,201,5.9,a352bb0a17ff +14.97,201,6.7,a352bb0a17ff +15.087,201,4.9,f6a858d7a20b +15.202,201,5.0,f6a858d7a20b +15.317,201,4.5,a352bb0a17ff +15.432,201,5.2,a352bb0a17ff +15.541,201,4.5,a352bb0a17ff +15.651,201,6.1,a352bb0a17ff +15.762,201,4.4,a352bb0a17ff +15.875,201,4.7,a352bb0a17ff +15.988,201,4.7,f6a858d7a20b +16.099,201,5.2,f6a858d7a20b +16.211,201,5.6,f6a858d7a20b +16.326,201,6.2,a352bb0a17ff +16.433,201,6.5,f6a858d7a20b +16.541,201,6.5,f6a858d7a20b +16.649,201,5.5,f6a858d7a20b +16.765,201,5.4,f6a858d7a20b +16.874,201,5.8,f6a858d7a20b +16.982,201,5.9,f6a858d7a20b +17.091,201,5.6,f6a858d7a20b +17.2,201,5.0,f6a858d7a20b +17.315,201,4.8,f6a858d7a20b +17.42,201,3.8,f6a858d7a20b +17.529,201,4.1,f6a858d7a20b +17.638,201,5.1,f6a858d7a20b +17.75,201,5.8,f6a858d7a20b +17.861,201,4.3,f6a858d7a20b +17.975,201,3.9,f6a858d7a20b +18.089,201,5.2,a352bb0a17ff +18.2,201,4.9,a352bb0a17ff +18.305,201,5.5,f6a858d7a20b +18.42,201,3.9,a352bb0a17ff +18.528,201,5.5,a352bb0a17ff +18.644,201,5.7,a352bb0a17ff +18.759,201,5.8,f6a858d7a20b +18.875,201,13.8,f6a858d7a20b +18.999,201,7.0,f6a858d7a20b +19.107,201,5.4,f6a858d7a20b +19.216,201,5.2,f6a858d7a20b +19.329,201,4.8,f6a858d7a20b +19.442,201,5.7,f6a858d7a20b +19.557,201,5.0,a352bb0a17ff +19.669,201,5.0,a352bb0a17ff +19.775,201,5.4,f6a858d7a20b +19.883,201,4.8,f6a858d7a20b +19.99,201,5.2,f6a858d7a20b +20.099,201,5.9,f6a858d7a20b +20.21,201,5.0,f6a858d7a20b +20.323,201,6.7,f6a858d7a20b +20.436,201,6.0,a352bb0a17ff +20.549,201,5.0,a352bb0a17ff +20.659,201,6.0,a352bb0a17ff +20.776,201,6.0,f6a858d7a20b +20.891,201,5.5,f6a858d7a20b +20.999,201,4.2,f6a858d7a20b +21.114,201,4.9,f6a858d7a20b +21.229,201,6.7,a352bb0a17ff +21.345,201,5.1,a352bb0a17ff +21.458,201,5.4,f6a858d7a20b +21.566,201,7.0,f6a858d7a20b +21.675,201,4.8,a352bb0a17ff +21.785,201,6.3,a352bb0a17ff +21.9,201,6.3,f6a858d7a20b +22.011,201,4.7,f6a858d7a20b +22.121,201,5.4,f6a858d7a20b +22.236,201,5.5,f6a858d7a20b +22.347,201,7.6,a352bb0a17ff +22.457,201,3.3,a352bb0a17ff +22.569,201,5.6,a352bb0a17ff +22.683,201,4.9,a352bb0a17ff +22.796,201,7.3,f6a858d7a20b +22.905,201,6.8,f6a858d7a20b +23.022,201,3.2,f6a858d7a20b +23.134,201,3.4,f6a858d7a20b +23.248,201,4.7,a352bb0a17ff +23.36,201,5.2,a352bb0a17ff +23.475,201,5.7,f6a858d7a20b +23.591,201,5.1,f6a858d7a20b +23.703,201,6.1,f6a858d7a20b +23.819,201,4.7,a352bb0a17ff +23.932,201,4.9,a352bb0a17ff +24.047,201,5.1,f6a858d7a20b +24.163,201,4.9,f6a858d7a20b +24.274,201,5.5,f6a858d7a20b +24.383,201,5.0,f6a858d7a20b +24.498,201,6.6,a352bb0a17ff +24.612,201,6.6,a352bb0a17ff +24.725,201,4.9,f6a858d7a20b +24.838,201,5.7,f6a858d7a20b +24.95,201,5.9,f6a858d7a20b +25.064,201,5.3,f6a858d7a20b +25.175,201,5.8,f6a858d7a20b +25.282,201,5.9,f6a858d7a20b +25.394,201,6.7,f6a858d7a20b +25.502,201,4.7,f6a858d7a20b +25.607,201,5.5,f6a858d7a20b +25.715,201,5.7,a352bb0a17ff +25.824,201,6.6,a352bb0a17ff +25.941,201,5.8,f6a858d7a20b +26.055,201,5.6,a352bb0a17ff +26.171,201,4.7,a352bb0a17ff +26.285,201,5.0,a352bb0a17ff +26.395,201,6.0,f6a858d7a20b +26.511,201,4.7,f6a858d7a20b +26.624,201,5.3,f6a858d7a20b +26.731,201,6.1,f6a858d7a20b +26.847,201,7.0,a352bb0a17ff +26.959,201,6.3,a352bb0a17ff +27.076,201,4.8,f6a858d7a20b +27.19,201,4.7,f6a858d7a20b +27.299,201,5.0,f6a858d7a20b +27.406,201,5.9,f6a858d7a20b +27.516,201,6.1,f6a858d7a20b +27.624,201,4.7,a352bb0a17ff +27.732,201,5.0,a352bb0a17ff +27.845,201,4.9,a352bb0a17ff +27.951,201,5.9,a352bb0a17ff +28.067,201,5.7,f6a858d7a20b +28.183,201,4.0,f6a858d7a20b +28.295,201,5.2,a352bb0a17ff +28.407,201,4.5,a352bb0a17ff +28.521,201,5.3,a352bb0a17ff +28.635,201,5.4,a352bb0a17ff +28.749,201,6.2,f6a858d7a20b +28.857,201,6.5,f6a858d7a20b +28.966,201,6.6,f6a858d7a20b +29.083,201,7.0,f6a858d7a20b +29.199,201,6.1,a352bb0a17ff +29.315,201,5.8,f6a858d7a20b +29.424,201,6.6,a352bb0a17ff +29.538,201,3.3,f6a858d7a20b +29.65,201,4.7,f6a858d7a20b +29.76,201,4.8,f6a858d7a20b +29.874,201,4.8,f6a858d7a20b +29.985,201,4.5,f6a858d7a20b +30.099,201,4.8,a352bb0a17ff +30.21,201,4.4,a352bb0a17ff +30.322,201,5.6,a352bb0a17ff +30.429,201,5.3,f6a858d7a20b +30.543,201,6.6,a352bb0a17ff +30.657,201,4.8,f6a858d7a20b +30.772,201,4.9,f6a858d7a20b +30.882,201,4.6,f6a858d7a20b +30.997,201,5.0,a352bb0a17ff +31.108,201,5.0,a352bb0a17ff +31.223,201,7.4,a352bb0a17ff +31.34,201,5.0,f6a858d7a20b +31.448,201,5.9,f6a858d7a20b +31.564,201,4.3,f6a858d7a20b +31.679,201,5.1,a352bb0a17ff +31.794,201,6.2,a352bb0a17ff +31.907,201,5.1,f6a858d7a20b +32.02,201,4.9,f6a858d7a20b +32.135,201,6.1,f6a858d7a20b +32.251,201,5.0,a352bb0a17ff +32.358,201,5.3,a352bb0a17ff +32.473,201,5.4,a352bb0a17ff +32.582,201,4.6,a352bb0a17ff +32.69,201,7.3,a352bb0a17ff +32.804,201,3.5,a352bb0a17ff +32.916,201,3.7,a352bb0a17ff +33.022,201,4.4,a352bb0a17ff +33.134,201,4.6,a352bb0a17ff +33.248,201,4.4,f6a858d7a20b +33.357,201,3.4,f6a858d7a20b +33.461,201,3.3,a352bb0a17ff +33.565,201,4.6,f6a858d7a20b +33.676,201,4.4,f6a858d7a20b +33.787,201,4.2,a352bb0a17ff +33.899,201,4.8,a352bb0a17ff +34.008,201,5.1,a352bb0a17ff +34.116,201,4.2,f6a858d7a20b +34.224,201,4.3,f6a858d7a20b +34.331,201,4.9,f6a858d7a20b +34.437,201,6.7,f6a858d7a20b +34.553,201,5.1,f6a858d7a20b +34.665,201,5.4,a352bb0a17ff +34.781,201,4.7,a352bb0a17ff +34.888,201,5.0,a352bb0a17ff +35.002,201,5.9,a352bb0a17ff +35.117,201,5.6,f6a858d7a20b +35.226,201,6.0,f6a858d7a20b +35.333,201,4.4,a352bb0a17ff +35.442,201,3.4,a352bb0a17ff +35.554,201,4.0,a352bb0a17ff +35.662,201,4.0,a352bb0a17ff +35.771,201,3.7,a352bb0a17ff +35.882,201,3.2,a352bb0a17ff +35.989,201,3.9,a352bb0a17ff +36.097,201,5.2,a352bb0a17ff +36.207,201,4.2,a352bb0a17ff +36.314,201,3.7,f6a858d7a20b +36.422,201,6.0,f6a858d7a20b +36.537,201,7.3,a352bb0a17ff +36.645,201,4.5,f6a858d7a20b +36.751,201,5.0,f6a858d7a20b +36.858,201,4.9,f6a858d7a20b +36.965,201,3.9,a352bb0a17ff +37.072,201,2.9,a352bb0a17ff +37.18,201,3.6,a352bb0a17ff +37.285,201,3.4,f6a858d7a20b +37.39,201,3.2,f6a858d7a20b +37.5,201,4.0,f6a858d7a20b +37.605,201,3.8,a352bb0a17ff +37.713,201,4.2,a352bb0a17ff +37.825,201,5.2,a352bb0a17ff +37.935,201,4.4,f6a858d7a20b +38.049,201,3.5,f6a858d7a20b +38.157,201,4.8,f6a858d7a20b +38.262,201,5.7,a352bb0a17ff +38.374,201,4.4,a352bb0a17ff +38.488,201,4.3,f6a858d7a20b +38.592,201,3.0,a352bb0a17ff +38.7,201,4.6,a352bb0a17ff +38.815,201,5.0,a352bb0a17ff +38.924,201,5.4,a352bb0a17ff +39.032,201,4.7,a352bb0a17ff +39.146,201,5.1,f6a858d7a20b +39.26,201,4.5,f6a858d7a20b +39.373,201,5.0,f6a858d7a20b +39.478,201,4.8,f6a858d7a20b +39.584,201,4.4,a352bb0a17ff +39.69,201,3.8,a352bb0a17ff +39.799,201,5.9,a352bb0a17ff +39.91,201,4.0,a352bb0a17ff +40.024,201,5.8,f6a858d7a20b +40.138,201,6.4,f6a858d7a20b +40.249,201,6.3,f6a858d7a20b +40.364,201,4.7,f6a858d7a20b +40.472,201,5.2,f6a858d7a20b +40.579,201,4.3,f6a858d7a20b +40.69,201,4.8,f6a858d7a20b +40.8,201,5.4,f6a858d7a20b +40.911,201,5.2,f6a858d7a20b +41.017,201,5.8,f6a858d7a20b +41.124,201,6.2,a352bb0a17ff +41.24,201,5.3,f6a858d7a20b +41.348,201,5.2,f6a858d7a20b +41.463,201,5.9,f6a858d7a20b +41.578,201,6.0,a352bb0a17ff +41.694,201,7.4,f6a858d7a20b +41.807,201,5.0,f6a858d7a20b +41.912,201,5.1,a352bb0a17ff +42.023,201,4.5,a352bb0a17ff +42.136,201,6.5,f6a858d7a20b +42.249,201,6.6,f6a858d7a20b +42.365,201,4.1,f6a858d7a20b +42.473,201,4.6,f6a858d7a20b +42.584,201,4.8,f6a858d7a20b +42.699,201,4.1,a352bb0a17ff +42.812,201,4.7,a352bb0a17ff +42.924,201,4.9,a352bb0a17ff +43.039,201,3.8,f6a858d7a20b +43.153,201,4.1,f6a858d7a20b +43.258,201,3.7,f6a858d7a20b +43.372,201,5.7,f6a858d7a20b +43.482,201,4.4,f6a858d7a20b +43.588,201,3.9,f6a858d7a20b +43.702,201,5.0,f6a858d7a20b +43.813,201,4.5,f6a858d7a20b +43.919,201,5.7,f6a858d7a20b +44.026,201,4.5,f6a858d7a20b +44.137,201,6.0,f6a858d7a20b +44.249,201,4.7,f6a858d7a20b +44.356,201,4.8,f6a858d7a20b +44.464,201,4.9,a352bb0a17ff +44.579,201,4.8,f6a858d7a20b +44.689,201,4.6,f6a858d7a20b +44.796,201,5.6,f6a858d7a20b +44.907,201,5.6,f6a858d7a20b +45.022,201,5.6,f6a858d7a20b +45.132,201,5.7,f6a858d7a20b +45.248,201,5.6,a352bb0a17ff +45.357,201,5.4,a352bb0a17ff +45.471,201,4.4,a352bb0a17ff +45.584,201,6.1,a352bb0a17ff +45.698,201,7.1,f6a858d7a20b +45.813,201,5.0,f6a858d7a20b +45.926,201,5.2,a352bb0a17ff +46.041,201,6.4,a352bb0a17ff +46.158,201,5.4,f6a858d7a20b +46.264,201,4.9,a352bb0a17ff +46.379,201,5.1,f6a858d7a20b +46.485,201,6.4,f6a858d7a20b +46.593,201,6.3,a352bb0a17ff +46.709,201,6.6,f6a858d7a20b +46.824,201,4.8,f6a858d7a20b +46.932,201,4.9,f6a858d7a20b +47.046,201,5.3,a352bb0a17ff +47.155,201,6.4,a352bb0a17ff +47.267,201,6.9,a352bb0a17ff +47.381,201,6.2,a352bb0a17ff +47.497,201,4.4,f6a858d7a20b +47.611,201,5.1,f6a858d7a20b +47.723,201,6.6,f6a858d7a20b +47.839,201,4.7,a352bb0a17ff +47.954,201,5.3,a352bb0a17ff +48.063,201,4.3,a352bb0a17ff +48.173,201,8.3,f6a858d7a20b +48.289,201,3.3,f6a858d7a20b +48.403,201,4.2,f6a858d7a20b +48.51,201,5.0,f6a858d7a20b +48.62,201,4.5,f6a858d7a20b +48.73,201,4.4,f6a858d7a20b +48.844,201,3.7,f6a858d7a20b +48.954,201,5.2,f6a858d7a20b +49.062,201,5.6,f6a858d7a20b +49.169,201,5.5,f6a858d7a20b +49.28,201,4.8,f6a858d7a20b +49.393,201,5.1,f6a858d7a20b +49.508,201,7.2,a352bb0a17ff +49.622,201,5.1,a352bb0a17ff +49.73,201,6.4,a352bb0a17ff +49.846,201,5.8,f6a858d7a20b +49.962,201,4.6,f6a858d7a20b +50.073,201,4.8,f6a858d7a20b +50.187,201,6.1,a352bb0a17ff +50.297,201,6.1,a352bb0a17ff +50.413,201,5.0,a352bb0a17ff +50.528,201,4.6,f6a858d7a20b +50.635,201,4.7,f6a858d7a20b +50.748,201,4.5,f6a858d7a20b +50.857,201,5.3,f6a858d7a20b +50.965,201,5.0,a352bb0a17ff +51.071,201,5.1,a352bb0a17ff +51.179,201,4.8,a352bb0a17ff +51.292,201,4.5,a352bb0a17ff +51.398,201,6.2,a352bb0a17ff +51.512,201,4.5,a352bb0a17ff +51.623,201,5.7,a352bb0a17ff +51.738,201,5.6,f6a858d7a20b +51.847,201,4.5,f6a858d7a20b +51.956,201,4.3,f6a858d7a20b +52.063,201,6.3,a352bb0a17ff +52.174,201,4.7,a352bb0a17ff +52.281,201,5.9,a352bb0a17ff +52.388,201,4.5,a352bb0a17ff +52.498,201,4.2,a352bb0a17ff +52.612,201,3.8,a352bb0a17ff +52.722,201,5.0,a352bb0a17ff +52.832,201,4.6,a352bb0a17ff +52.94,201,5.0,a352bb0a17ff +53.05,201,5.2,a352bb0a17ff +53.165,201,5.3,f6a858d7a20b +53.273,201,5.8,a352bb0a17ff +53.389,201,3.7,f6a858d7a20b +53.498,201,3.1,f6a858d7a20b +53.611,201,5.7,f6a858d7a20b +53.723,201,4.8,f6a858d7a20b +53.833,201,5.5,f6a858d7a20b +53.948,201,4.3,a352bb0a17ff +54.056,201,4.3,a352bb0a17ff +54.164,201,4.6,f6a858d7a20b +54.271,201,5.6,f6a858d7a20b +54.385,201,5.1,f6a858d7a20b +54.499,201,4.9,a352bb0a17ff +54.614,201,4.8,a352bb0a17ff +54.722,201,5.0,a352bb0a17ff +54.836,201,5.0,f6a858d7a20b +54.948,201,6.4,f6a858d7a20b +55.058,201,5.9,f6a858d7a20b +55.166,201,5.4,f6a858d7a20b +55.273,201,5.1,a352bb0a17ff +55.38,201,4.8,a352bb0a17ff +55.495,201,5.8,f6a858d7a20b +55.602,201,5.4,a352bb0a17ff +55.715,201,6.9,f6a858d7a20b +55.823,201,4.0,a352bb0a17ff +55.937,201,4.4,f6a858d7a20b +56.047,201,5.0,f6a858d7a20b +56.153,201,6.6,a352bb0a17ff +56.268,201,5.2,f6a858d7a20b +56.383,201,4.8,f6a858d7a20b +56.494,201,4.3,f6a858d7a20b +56.602,201,4.2,f6a858d7a20b +56.714,201,5.4,f6a858d7a20b +56.825,201,5.2,a352bb0a17ff +56.933,201,4.9,f6a858d7a20b +57.046,201,5.0,a352bb0a17ff +57.157,201,6.7,a352bb0a17ff +57.273,201,4.6,a352bb0a17ff +57.388,201,4.8,f6a858d7a20b +57.498,201,5.3,f6a858d7a20b +57.613,201,6.7,f6a858d7a20b +57.728,201,5.3,a352bb0a17ff +57.843,201,4.9,a352bb0a17ff +57.959,201,5.1,f6a858d7a20b +58.073,201,4.4,f6a858d7a20b +58.181,201,5.2,f6a858d7a20b +58.296,201,4.1,a352bb0a17ff +58.406,201,4.7,f6a858d7a20b +58.511,201,3.6,f6a858d7a20b +58.623,201,3.2,f6a858d7a20b +58.731,201,3.8,f6a858d7a20b +58.845,201,4.6,a352bb0a17ff +58.956,201,5.2,a352bb0a17ff +59.063,201,4.9,f6a858d7a20b +59.172,201,3.1,f6a858d7a20b +59.283,201,4.6,f6a858d7a20b +59.388,201,4.6,f6a858d7a20b +59.498,201,6.3,f6a858d7a20b +59.606,201,5.3,f6a858d7a20b +59.716,201,4.9,f6a858d7a20b +59.831,201,5.7,f6a858d7a20b +59.944,201,6.2,f6a858d7a20b +60.059,201,3.9,a352bb0a17ff +60.164,201,4.3,a352bb0a17ff +60.275,201,4.7,a352bb0a17ff +60.389,201,4.7,a352bb0a17ff +60.497,201,6.6,a352bb0a17ff +60.609,201,5.9,a352bb0a17ff +60.725,201,4.6,f6a858d7a20b +60.838,201,4.8,f6a858d7a20b +60.946,201,5.5,f6a858d7a20b +61.056,201,5.2,f6a858d7a20b +61.17,201,5.5,f6a858d7a20b +61.284,201,5.8,a352bb0a17ff +61.4,201,4.7,a352bb0a17ff +61.515,201,5.2,f6a858d7a20b +61.628,201,5.8,f6a858d7a20b +61.735,201,4.9,f6a858d7a20b +61.849,201,4.8,f6a858d7a20b +61.957,201,5.0,f6a858d7a20b +62.064,201,5.6,a352bb0a17ff +62.174,201,5.3,f6a858d7a20b +62.29,201,4.4,f6a858d7a20b +62.398,201,4.7,f6a858d7a20b +62.506,201,3.7,f6a858d7a20b +62.613,201,3.3,a352bb0a17ff +62.721,201,3.8,a352bb0a17ff +62.835,201,3.0,f6a858d7a20b +62.942,201,5.1,a352bb0a17ff +63.053,201,4.9,a352bb0a17ff +63.16,201,4.7,a352bb0a17ff +63.267,201,3.7,a352bb0a17ff +63.374,201,4.3,f6a858d7a20b +63.479,201,3.4,f6a858d7a20b +63.584,201,4.0,a352bb0a17ff +63.698,201,4.5,f6a858d7a20b +63.813,201,3.3,f6a858d7a20b +63.922,201,3.6,f6a858d7a20b +64.031,201,3.2,f6a858d7a20b +64.14,201,3.8,f6a858d7a20b +64.254,201,3.9,f6a858d7a20b +64.365,201,4.7,a352bb0a17ff +64.476,201,4.1,a352bb0a17ff +64.581,201,3.0,f6a858d7a20b +64.689,201,5.1,f6a858d7a20b +64.804,201,3.9,f6a858d7a20b +64.916,201,4.1,a352bb0a17ff +65.031,201,4.0,a352bb0a17ff +65.135,201,4.7,a352bb0a17ff +65.247,201,3.9,a352bb0a17ff +65.355,201,4.1,a352bb0a17ff +65.461,201,5.5,f6a858d7a20b +65.569,201,4.3,f6a858d7a20b +65.678,201,5.0,f6a858d7a20b +65.793,201,5.6,f6a858d7a20b +65.902,201,6.3,f6a858d7a20b +66.018,201,6.3,a352bb0a17ff +66.127,201,6.1,a352bb0a17ff +66.234,201,7.3,a352bb0a17ff +66.346,201,4.3,a352bb0a17ff +66.459,201,5.5,a352bb0a17ff +66.571,201,4.6,a352bb0a17ff +66.685,201,4.8,a352bb0a17ff +66.79,201,4.7,a352bb0a17ff +66.896,201,6.2,a352bb0a17ff +67.006,201,6.4,a352bb0a17ff +67.113,201,5.2,f6a858d7a20b +67.221,201,6.5,f6a858d7a20b +67.333,201,6.9,f6a858d7a20b +67.447,201,4.3,a352bb0a17ff +67.559,201,4.7,a352bb0a17ff +67.674,201,5.0,a352bb0a17ff +67.789,201,4.8,f6a858d7a20b +67.904,201,5.0,f6a858d7a20b +68.019,201,5.2,a352bb0a17ff +68.13,201,4.8,a352bb0a17ff +68.237,201,5.7,a352bb0a17ff +68.35,201,6.3,a352bb0a17ff +68.467,201,5.7,f6a858d7a20b +68.578,201,5.8,f6a858d7a20b +68.694,201,5.6,a352bb0a17ff +68.801,201,4.2,f6a858d7a20b +68.913,201,3.6,f6a858d7a20b +69.017,201,3.9,f6a858d7a20b +69.127,201,6.3,f6a858d7a20b +69.243,201,5.7,f6a858d7a20b +69.356,201,4.7,a352bb0a17ff +69.464,201,3.5,f6a858d7a20b +69.57,201,4.6,f6a858d7a20b +69.684,201,5.4,a352bb0a17ff +69.8,201,4.9,a352bb0a17ff +69.914,201,4.5,f6a858d7a20b +70.029,201,4.8,f6a858d7a20b +70.138,201,4.7,f6a858d7a20b +70.252,201,4.8,f6a858d7a20b +70.363,201,6.1,a352bb0a17ff +70.472,201,4.1,f6a858d7a20b +70.586,201,4.8,a352bb0a17ff +70.693,201,4.5,f6a858d7a20b +70.808,201,5.4,a352bb0a17ff +70.922,201,7.1,f6a858d7a20b +71.033,201,4.6,a352bb0a17ff +71.139,201,4.5,a352bb0a17ff +71.254,201,4.9,a352bb0a17ff +71.364,201,5.2,f6a858d7a20b +71.472,201,5.8,a352bb0a17ff +71.58,201,4.7,a352bb0a17ff +71.69,201,5.2,a352bb0a17ff +71.805,201,5.5,f6a858d7a20b +71.912,201,5.4,a352bb0a17ff +72.023,201,5.0,a352bb0a17ff +72.135,201,5.3,f6a858d7a20b +72.247,201,5.6,f6a858d7a20b +72.356,201,5.0,f6a858d7a20b +72.463,201,4.6,a352bb0a17ff +72.577,201,4.6,f6a858d7a20b +72.69,201,4.9,f6a858d7a20b +72.805,201,4.1,f6a858d7a20b +72.915,201,3.2,a352bb0a17ff +73.023,201,4.5,f6a858d7a20b +73.131,201,5.2,f6a858d7a20b +73.246,201,4.9,a352bb0a17ff +73.351,201,4.8,f6a858d7a20b +73.464,201,6.2,a352bb0a17ff +73.576,201,5.3,a352bb0a17ff +73.691,201,5.4,a352bb0a17ff +73.799,201,5.1,a352bb0a17ff +73.904,201,5.5,a352bb0a17ff +74.018,201,4.3,a352bb0a17ff +74.13,201,3.5,a352bb0a17ff +74.239,201,5.0,a352bb0a17ff +74.346,201,4.3,a352bb0a17ff +74.461,201,4.5,a352bb0a17ff +74.575,201,3.3,f6a858d7a20b +74.688,201,4.0,f6a858d7a20b +74.801,201,4.4,f6a858d7a20b +74.916,201,3.9,a352bb0a17ff +75.02,201,5.1,f6a858d7a20b +75.135,201,7.1,a352bb0a17ff +75.246,201,13.6,f6a858d7a20b +75.366,201,11.8,a352bb0a17ff +75.484,201,8.2,f6a858d7a20b diff --git a/evidence/rolling/result.json b/evidence/rolling/result.json index e3e6b869..7c4d2a4a 100644 --- a/evidence/rolling/result.json +++ b/evidence/rolling/result.json @@ -1,30 +1,38 @@ -{ +{"date": "2026-11-05T19:21:07.986361+00:00", "commit": "252e3e3ffeea5004cfd9dc342e686af1417a77a2", "label": "action", "action": [ "scripts/rolling-update.sh" ], "action_exit": 0, - "action_seconds": 12.62, + "action_seconds": 10.734, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 2, + "lb_samples_missing": 0, "detection_seconds": null, "rejoin_seconds": null, "probe": { - "attempts": 919, + "attempts": 913, "failed_attempts": 0, "first_failure_s": null, + "first_success_s": 0.055, "recovery_s": 0.0, "instances_before": { - "6c2a02267340": 110, - "158dfc1bd95d": 125 + "5bbf259ff136": 120, + "7cfd91ddf780": 113 }, "instances_during_outage": {}, "instances_after": { - "c2f5d82dee7c": 310, - "65d10d346f4e": 260 + "a352bb0a17ff": 255, + "f6a858d7a20b": 328 } }, + "history": { + "confirmed": 913, + "stored": 913, + "exact_match": true, + "unique": true + }, "lb_counters": { "wretr": 0, "wredis": 0, @@ -38,16 +46,19 @@ "ok_rps": 100.01, "failed_ratio": 0, "dropped_iterations": 0, - "p50_ms": 1.34, - "p95_ms": 2.21, - "p99_ms": 6.86, + "p50_ms": 3.01, + "p95_ms": 3.89, + "p99_ms": 4.53, "exit": 0, "rate": 100 }, "checks": { "action_ok": true, "load_errors_ok": true, + "load_delivered": true, + "lb_observed": true, "write_recovery_ok": true, + "history_preserved": true, "capacity_kept": true, "all_replaced": true }, diff --git a/openspec/changes/2026-11-05-adopt-sqlalchemy-alembic/design.md b/openspec/changes/2026-11-05-adopt-sqlalchemy-alembic/design.md index 6ef455e8..a0c58756 100644 --- a/openspec/changes/2026-11-05-adopt-sqlalchemy-alembic/design.md +++ b/openspec/changes/2026-11-05-adopt-sqlalchemy-alembic/design.md @@ -129,5 +129,7 @@ top-операторы SQLAlchemy, разница в RPS/латентности, неизвестную ревизию, повторный и конкурентный запуск. Пул проверяется на rollback, таймаутах и отмене запроса. Уровень «3» подтверждён этапом grade3 на `0a1f48bd73e5780c732356880c23fdb476bcff98`; обнаруженная регрессия закрытых соединений исправлена в `fix-lab3-postgres-pool-recovery`. Проверка соединения добавляет обращение к БД при выдаче из пула. -Уровень «4» и измерения трека 1 остаются следующими этапами; прежние цифры не подтверждают -производительность SQLAlchemy. Change остаётся активным до этих прогонов и обновления отчёта. +Уровень «4» повторно подтверждён на `252e3e3`: detection, каждый failover и два rolling update под 100 rps, +лимиты и однократный Alembic. Доказательства — grade4-логи и таблица 4.x REPORT.md. +Измерения трека 1 остаются следующему этапу; прежние цифры raw asyncpg не подтверждают +производительность SQLAlchemy. Change остаётся активным до трека и итогового обновления отчёта. diff --git a/openspec/changes/2026-11-05-adopt-sqlalchemy-alembic/tasks.md b/openspec/changes/2026-11-05-adopt-sqlalchemy-alembic/tasks.md index ed2930bc..bdeee2f2 100644 --- a/openspec/changes/2026-11-05-adopt-sqlalchemy-alembic/tasks.md +++ b/openspec/changes/2026-11-05-adopt-sqlalchemy-alembic/tasks.md @@ -23,7 +23,7 @@ - [x] 4.2 Стенд из чистого состояния: `make up` (миграции один раз), `make test`, `make lb-check`, `make retry-check`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-faults`, `make restore` - [x] 4.3 `evidence-storage.sh` прогон: `evidence/{schema,migrations,kdf,secrets}.txt` на новом коде; свежие `smoke.txt`, `student-tests.txt`, `checks.txt` -- [ ] 4.4 Этап grade4: `make level4` под нагрузкой на SQLAlchemy, обновить соответствующие доказательства +- [x] 4.4 Этап grade4: `make level4` под нагрузкой на SQLAlchemy, обновить соответствующие доказательства ## 5. Трек 1 и отчёт @@ -69,3 +69,20 @@ Задача 4.2 разделена по последовательным этапам: проверки уровня «3» завершены, `make level4` перенесён в 4.4. Выполнено 13 из 17 задач. Задачи 4.4 и 5.1–5.3 остаются следующим этапам; новые RPS/латентности SQLAlchemy с pre-ping пока не измерены. Change не архивируется. + +## Результат этапа grade4 — 2026-11-05 + +Уровень «4» повторно подтверждён на `252e3e3ffeea5004cfd9dc342e686af1417a77a2`: +все цели `make level4` выполнены по отдельности, rolling также с `FROM_REV=7fc911c`. +12/12 detection PASS; каждая реплика остановлена под 100 rps (по 6001 запросу, 0 ошибок), +оба rolling update — по 9001 запросу, 0 ошибок, минимум 2 UP. Полная история зонда сверена +с подтверждёнными Message; 645/642 сообщений в failover и по 913 в rolling без потерь/дублей. +Реальные CPU/RAM-лимиты, порты, secrets/volumes/сети проверены; SIGTERM и цепочка таймаутов — 3 passed. +Повтор Alembic сохранил head и все счётчики строк. Публичные smoke/persistence/failover — PASS; +persistence повторён со снимками ID/StartedAt до/после. Полный набор — 285 passed, 12 deselected. + +Усиление нагрузочного зонда оформлено отдельным `verify-lab3-grade4-recovery`. +Неуспешные промежуточные проверки и итоговые результаты перечислены в evidence/README.md. +REPORT обновлён; публикаций и Telegram не было, контейнеры остановлены, volumes сохранены. +Выполнено 14 из 17 задач исходного change; 5.1–5.3 остаются этапу трека «5». +Миграции в rolling — no-op на совместимой существующей схеме; произвольный DDL этим не доказан. diff --git a/openspec/changes/verify-lab3-grade4-recovery/.openspec.yaml b/openspec/changes/archive/2026-11-05-verify-lab3-grade4-recovery/.openspec.yaml similarity index 57% rename from openspec/changes/verify-lab3-grade4-recovery/.openspec.yaml rename to openspec/changes/archive/2026-11-05-verify-lab3-grade4-recovery/.openspec.yaml index 1491b921..05b95809 100644 --- a/openspec/changes/verify-lab3-grade4-recovery/.openspec.yaml +++ b/openspec/changes/archive/2026-11-05-verify-lab3-grade4-recovery/.openspec.yaml @@ -1,2 +1,6 @@ schema: spec-driven created: 2026-11-05 + +archived: 2026-11-05 + +lab: 3 diff --git a/openspec/changes/verify-lab3-grade4-recovery/design.md b/openspec/changes/archive/2026-11-05-verify-lab3-grade4-recovery/design.md similarity index 100% rename from openspec/changes/verify-lab3-grade4-recovery/design.md rename to openspec/changes/archive/2026-11-05-verify-lab3-grade4-recovery/design.md diff --git a/openspec/changes/verify-lab3-grade4-recovery/proposal.md b/openspec/changes/archive/2026-11-05-verify-lab3-grade4-recovery/proposal.md similarity index 100% rename from openspec/changes/verify-lab3-grade4-recovery/proposal.md rename to openspec/changes/archive/2026-11-05-verify-lab3-grade4-recovery/proposal.md diff --git a/openspec/changes/verify-lab3-grade4-recovery/specs/lab3-failover-operations/spec.md b/openspec/changes/archive/2026-11-05-verify-lab3-grade4-recovery/specs/lab3-failover-operations/spec.md similarity index 100% rename from openspec/changes/verify-lab3-grade4-recovery/specs/lab3-failover-operations/spec.md rename to openspec/changes/archive/2026-11-05-verify-lab3-grade4-recovery/specs/lab3-failover-operations/spec.md diff --git a/openspec/changes/verify-lab3-grade4-recovery/tasks.md b/openspec/changes/archive/2026-11-05-verify-lab3-grade4-recovery/tasks.md similarity index 66% rename from openspec/changes/verify-lab3-grade4-recovery/tasks.md rename to openspec/changes/archive/2026-11-05-verify-lab3-grade4-recovery/tasks.md index 8d7af8fa..ed607aa6 100644 --- a/openspec/changes/verify-lab3-grade4-recovery/tasks.md +++ b/openspec/changes/archive/2026-11-05-verify-lab3-grade4-recovery/tasks.md @@ -9,12 +9,16 @@ ## 3. Стенд уровня «4» -- [ ] 3.1 Поднять отдельный проект на 18083/23083, записать версии, лимиты, readiness/SIGTERM и 20 гонок ключа. -- [ ] 3.2 Выполнить три серии detection и поочерёдный failover каждой реплики под 100 rps, сохранить результаты и реальные контейнеры. -- [ ] 3.3 Выполнить rolling update текущей сборкой и с предыдущей SQLAlchemy-версии; проверить один migrate, Alembic head и повтор без изменения данных. -- [ ] 3.4 Повторить публичные smoke/persistence/failover, остановить свой стенд с сохранением volumes. +- [x] 3.1 Поднять отдельный проект на 18083/23083, записать версии, лимиты, readiness/SIGTERM и 20 гонок ключа. +- [x] 3.2 Выполнить три серии detection и поочерёдный failover каждой реплики под 100 rps, сохранить результаты и реальные контейнеры. +- [x] 3.3 Выполнить rolling update текущей сборкой и с предыдущей SQLAlchemy-версии; проверить один migrate, Alembic head и повтор без изменения данных. +- [x] 3.4 Повторить публичные smoke/persistence/failover, остановить свой стенд с сохранением volumes. ## 4. Доказательства и завершение -- [ ] 4.1 Обновить REPORT, индекс evidence и задачу 4.4 исходного change; трек «5» оставить незавершённым. -- [ ] 4.2 `make check`, `make validate`, `make lint`, `make student-test`, `uv lock --check`, `openspec validate --all --strict`; синхронизировать спецификацию и архивировать этот change, закоммитить. +- [x] 4.1 Обновить REPORT, индекс evidence и задачу 4.4 исходного change; трек «5» оставить незавершённым. +- [x] 4.2 `make check`, `make validate`, `make lint`, `make student-test`, `uv lock --check`, `openspec validate --all --strict`; синхронизировать спецификацию и архивировать этот change, закоммитить. + +Результат: уровень «4» подтверждён на 252e3e3; таблица 4.x REPORT.md и evidence/grade4-*.txt. +Полный набор: 285 passed, 12 deselected; check/validate/lint/lock/OpenSpec — PASS. +Контейнеры остановлены, volumes сохранены; промежуточные неуспехи перечислены в evidence/README.md. diff --git a/openspec/specs/lab3-failover-operations/spec.md b/openspec/specs/lab3-failover-operations/spec.md index d0950f06..fea74fb4 100644 --- a/openspec/specs/lab3-failover-operations/spec.md +++ b/openspec/specs/lab3-failover-operations/spec.md @@ -81,4 +81,28 @@ #### Scenario: Миграции один раз - **WHEN** идёт поочерёдная замена -- **THEN** `migrate` запускается один раз до замены первой реплики, реплики схему не меняют, а повторный запуск сообщает `up to date` +- **THEN** `migrate` запускается один раз до замены первой реплики, реплики схему не меняют, а повторный `alembic upgrade head` завершается кодом 0 без новых ревизий и изменения данных + +### Requirement: Достоверность нагрузочного доказательства + +Для критериев уровня «4» README нагрузочный зонд SHALL измерять восстановление по завершению +первого успешного POST после последней ошибки, а не по времени самой ошибки. Первый успешный +POST после начала действия SHALL завершиться не позже 10 с. Отсутствие успешного ответа после +ошибки SHALL приводить к неуспешной проверке. После нагрузки полная история отдельного чата зонда +SHALL совпадать с подтверждёнными телами Message, без пропусков и дублей. Нулевое число запросов, +пропущенные итерации нагрузки и отсутствие наблюдений LB SHALL NOT давать PASS. + +#### Scenario: Ошибка не подтверждает восстановление + +- **WHEN** последняя попытка записи завершилась ошибкой либо успешный ответ после неё пришёл позже 10 с от начала действия +- **THEN** проверка восстановления завершается неуспешно, даже если сама ошибка возникла раньше 10 с + +#### Scenario: История после действия + +- **WHEN** поочерёдный отказ или rolling update завершён под нагрузкой +- **THEN** зонд читает все страницы своего чата и сравнивает все поля с ответами 201; пропуск, дубль или изменение сообщения дают неуспех + +#### Scenario: Неполная нагрузка или статистика + +- **WHEN** генератор не выполнил запросов, пропустил итерации либо опрос LB не получил статистику +- **THEN** прогон не подтверждает сохранение нагрузки и ёмкости и завершается неуспешно