diff --git a/AGENTS.md b/AGENTS.md index 315719eb..7b5fb908 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,11 @@ make test # публичный smoke через LB (BASE_U make test-persistence ACTION_SCRIPT=scripts/restart.sh # рестарт lb, api, postgres, valkey с сохранением volumes make test-failover ACTION_SCRIPT=scripts/stop-one.sh # остановка реплики TARGET_INSTANCE_ID под запросами make restore # вернуть остановленные реплики и дождаться их в ротации LB -make lb-check # ≥2 X-Instance-Id на одном токене и гонка одного Idempotency-Key +make lb-check # ≥2 X-Instance-Id на одном токене и 20 гонок одного Idempotency-Key +make limits # уровень 4: лимиты, порты, секреты, volumes, сети против политики scripts/limits.py +make detection # уровень 4: время обнаружения SIGTERM/kill/pause/PostgreSQL против заявленного максимума +make failover-each # уровень 4: каждая реплика по очереди под k6 (RATE=100) → evidence/failover-each +make rolling-update [FROM_REV=<коммит>] # уровень 4: замена реплик по одной под k6, migrate один раз make down # остановка без удаления данных; make reset-data — намеренное удаление volumes ``` diff --git a/REPORT.md b/REPORT.md index 5ab25cfb..e80cfdb0 100644 --- a/REPORT.md +++ b/REPORT.md @@ -5,7 +5,8 @@ - Commit/PR: ветка `solution/bpi2303-kostylev`, PR в репозиторий преподавателя - Использованные источники, библиотеки и помощь ИИ: FastAPI, gunicorn + uvicorn-worker, dishka, pydantic, structlog, asyncpg, argon2-cffi, redis-py, pytest, ruff, ty (версии — `evidence/environment.txt` и `uv.lock`); - HAProxy 3.2 Configuration Manual (health checks, resolvers, `retry-on`, `http-error`), документация Docker + HAProxy 3.2 Configuration Manual (health checks, resolvers, `retry-on`, `http-error`, `init-state`), Grafana k6 + (`constant-arrival-rate`, `handleSummary`), uvicorn и gunicorn (порядок graceful shutdown), документация Docker Compose (`deploy.replicas`, `depends_on`, secrets), PostgreSQL (`SELECT … FOR UPDATE`, `INSERT … ON CONFLICT`, уровни изоляции), IETF draft «The Idempotency-Key HTTP Header Field». Решение перенесено из моей лабы 2 и выполнено с помощью Claude Code; код, стенд и отказы проверены прогонами, перечисленными ниже. @@ -42,7 +43,7 @@ make reset-data # НАМЕРЕННОЕ удаление учебных да ```sh make install # uv sync --group dev (Python 3.14 и зависимости из uv.lock) -make lint # ruff check + ruff format --check (src, student-tests, scripts/probe.py) + ty check +make lint # ruff check + ruff format --check (src, student-tests, scripts/probe.py, scripts/limits.py) + ty check make student-test # собственные тесты; поднимают одноразовые PostgreSQL и Valkey того же образа в Docker make check # офлайн-проверка skeleton make validate PYTHON=.venv/bin/python # полная валидация OpenAPI @@ -53,6 +54,20 @@ make stack-evidence # сервисы, порты, hostname ↔ X-Instanc make storage-evidence # \d+ схемы, повтор миграций, префикс KDF, правила ignore ``` +Проверки уровня «4» — по одной команде на пункт рубрики, каждая пишет файл в `evidence/` и завершается +ненулевым кодом при нарушении (`make level4` — все подряд, ≈8 минут): + +```sh +make limits # лимиты, порты, секреты, volumes, сети по политике → evidence/limits.txt +make lb-check # 40 GET одним токеном + 20 раундов × 8 параллельных POST с одним ключом +make detection # время обнаружения: SIGTERM, kill, pause, остановка PostgreSQL → evidence/detection/ +make failover-each # каждая реплика: остановка, 10 с без неё, возврат — под k6 100 rps +make rolling-update FROM_REV=8d28d45 # стенд на версии уровня «3», затем замена на текущую под нагрузкой +make rolling-update # замена текущей сборкой (эквивалентная поочерёдная замена) +``` + +Генератор нагрузки — сервис `k6` под профилем `bench` (с `make up` не стартует); `RATE` задаёт интенсивность. + **Версии и стенд:** Python 3.14.7 в образе `python:3.14-slim-trixie@sha256:51dafde8…`, `haproxy:3.2.26-alpine@sha256:58938405…`, `postgres:17.11-alpine@sha256:266d3c25…`, `valkey/valkey:8.1.11-alpine@sha256:2602ace2…`, uv 0.9.7 по digest, Docker 28.3.2, Compose 2.38.2. Машина — @@ -110,7 +125,7 @@ message_id, created_at)`, PK `(chat_id, user_id, key)` — область «по | --- | --- | --- | | Обнаружение | `server-template api 8 api:8080`, `resolvers docker` (127.0.0.11), `init-addr none` | остановленная реплика пропадает из DNS, новая появляется без перезапуска LB и без `docker.sock` | | Алгоритм | `balance roundrobin`, без `stick` | каждое новое соединение — следующая реплика; сессия общая, привязка не нужна | -| Проверка | `GET /health/ready`, `inter 1s fastinter 500ms fall 2 rise 2`, `timeout check 1s` | реплика без БД или в остановке выходит из ротации | +| Проверка | `GET /health/ready`, `inter 1s fastinter 500ms downinter 2s fall 2 rise 2`, `timeout check 1s`, `init-state fully-down` | реплика без БД или в остановке выходит из ротации; вернувшаяся получает трафик только после проверок | | Повторы | `retries 2`, `redispatch 1`, `retry-on conn-failure empty-response`; `disable-l7-retry` всем, кроме `GET` и `POST …/messages` с ключом | отказ соединения повторяется всегда (запрос не ушёл); после отправки — только безопасное к повтору; `POST /users`, `/chats`, вход не повторяются (`contracts/README.md`) | | Соединения к API | `option http-server-close` | нет простаивающих keep-alive, за которые гонится остановка реплики | | Таймауты | `connect 500ms`, `server 10s` > `REQUEST_TIMEOUT_SECONDS=8` > `statement_timeout` 2 с; `queue 3s`, `maxconn 64` на сервер | цепочка клиент > LB > приложение > БД | @@ -118,12 +133,53 @@ message_id, created_at)`, PK `(chat_id, user_id, key)` — область «по | `X-Request-Id` | непустой входящий до 200 символов пробрасывается, иначе uuid4 | приложение эхо-возвращает его | | Остановка | `SIGUSR1`, `hard-stop-after 5s` | мягкий стоп внутри `stop_grace_period` | -**Остановка и процессы.** `SHUTDOWN_TIMEOUT_SECONDS=5` (gunicorn `graceful_timeout`, uvicorn -`timeout_graceful_shutdown`) < `stop_grace_period: 8s`; `stop-one.sh` останавливает `docker stop -t 6`. По -`SIGTERM` gunicorn закрывает listener — LB получает отказ соединения и повторяет запрос на соседней реплике. +**Остановка и процессы (уровень «4»).** По `SIGTERM` реплика сначала выходит из ротации и только потом +перестаёт принимать соединения. Каждый воркер запускает `DrainingServer` (наследник `uvicorn.Server`, +`src/messenger/gunicorn_conf.py`): после сигнала он поднимает флаг, `/health/ready` отвечает `503`, `/health/live` +— `200`, запросы обслуживаются как обычно, и так `DRAIN_SECONDS=2` — дольше худшего обнаружения `503` +балансировщиком (1,5 с). Затем uvicorn закрывает listener и дообрабатывает принятые запросы ещё +`SHUTDOWN_TIMEOUT_SECONDS − DRAIN_SECONDS` = 3 с; весь срок — gunicorn `graceful_timeout` 5 с < `docker stop -t 6` +хука < `stop_grace_period: 8s` < 10 с. Без окна (уровень «3») gunicorn закрывал listener сразу, и запрос, уже +стоявший в очереди `accept`, LB видел как обрыв после отправки: для `GET` и сообщений с ключом это повтор, для +`POST /users`, `/chats` и входа — `502`. Разница видна в счётчиках LB: rolling update со старой версии — +`wredis` 54, `eresp` 2, с новой — 0 (`evidence/rolling*/result.json`). Хук `stop-one.sh` теперь идёт ≈2,7 с. На реплику `cpus: 1`, `512m`, 2 воркера, пул 8: 2 × 2 × 8 = 32 соединения < `max_connections=200`. API работает `read_only` с `tmpfs /tmp`. +**Проверки, период и максимум обнаружения.** `GET /health/ready` раз в `inter 1s`; после первого провала — +`fastinter 500ms`; `fall 2` провала — реплика исключена; `rise 2` успеха — возвращена; `timeout check 1s`; +у `DOWN`-реплики — `downinter 2s`. Readiness пингует PostgreSQL за ≤0,8 с (0,4 с на соединение из пула и 0,4 с +на `SELECT 1`), то есть отвечает явным `503` раньше таймаута проверки. `init-state fully-down`: сервер, который +стартовал, появился через DNS или вернулся из `MAINT (resolution)`, получает трафик только после `rise 2` +проверок (без этого HAProxy делал его `UP/READY` сразу по DNS-ответу, и запросы уходили в ещё не слушающий +gunicorn — первый прогон `failover-each` показал `wredis` 19 и 49 без ошибок). + +| Отказ | Как замечает LB | Максимум обнаружения | Замер (`make detection`) | +| --- | --- | --- | --- | +| `SIGTERM` (`docker stop`, `stop-one.sh`) | readiness `503` в окне вывода | `inter + (fall−1)·fastinter` = 1,5 с, и раньше закрытия порта (2 с) | 0,71 с; процесс вышел через 2,49 с | +| процесс убит (`docker kill`) | отказ соединения, DNS | 1,5 с | 0,67 с | +| процесс завис (`docker pause`) | таймаут проверки | `inter + (fall−1)·fastinter + fall·timeout check` = 3,5 с | 2,76 с | +| PostgreSQL остановлен | readiness всех реплик `503` | 1,5 с до 0 `UP`; LB сам отвечает `503` | 1,23 с; `/health/live` через LB → `503 unavailable` без `X-Instance-Id` | + +Допуск замера — 0,25 с (задержка docker CLI до сигнала и опрос статистики LB раз в 50 мс); время считается +от запуска команды отказа. Возврат в ротацию после `restore` — 2,0–3,1 с. + +**Цепочка таймаутов** (проверяется тестом `student-tests/unit/test_timeouts.py` по `compose.yaml`, +`haproxy.cfg` и `scripts/stop-one.sh`, а не только таблицей): + +| Звено | Значение | Условие | +| --- | --- | --- | +| LB `timeout server` | 10 с | > `REQUEST_TIMEOUT_SECONDS` | +| `REQUEST_TIMEOUT_SECONDS` | 8 с | > ожидание соединения + оператор БД | +| `DB_ACQUIRE_TIMEOUT_SECONDS` + `DB_COMMAND_TIMEOUT_SECONDS` (`statement_timeout` = `lock_timeout`) | 1 + 2 с | ≤ дообработки 3 с после окна | +| LB `timeout check` | 1 с | > readiness 0,8 с | +| `DRAIN_SECONDS` | 2 с | > обнаружения 1,5 с | +| `SHUTDOWN_TIMEOUT_SECONDS` (gunicorn `graceful_timeout`) | 5 с | > окна; < `STOP_TIMEOUT` хука | +| `STOP_TIMEOUT` хука `stop-one.sh` | 6 с | < `stop_grace_period` 8 с < 10 с | +| LB `timeout connect` / `queue` | 0,5 с / 3 с | отказ соединения повторяется на соседа сразу | +| LB `hard-stop-after` | 5 с | < `stop_grace_period` LB 8 с | +| клиенты: resilience / smoke / k6 | 2 / 10 / 10 с | — | + **Хуки.** `scripts/stop-one.sh` — контейнер `api` с `.Config.Hostname == TARGET_INSTANCE_ID` → `docker stop -t 6`, реплика остаётся остановленной, неизвестный ID → код 3 без изменений. `scripts/restart.sh` — `stop lb api`, `restart postgres valkey`, `up -d --no-recreate --wait` (те же контейнеры, повторный `migrate`), volumes не @@ -139,11 +195,13 @@ message_id, created_at)`, PK `(chat_id, user_id, key)` — область «по | postgres | `postgres` | `postgres:17.11-alpine@sha256:266d3c25…`, uid 70, `pg_stat_statements` | `pg_isready` | `pgdata:/var/lib/postgresql/data` | `backend` | secret `postgres_password` через `POSTGRES_PASSWORD_FILE` | | сессии (кэш) | `valkey` | `valkey/valkey:8.1.11-alpine@sha256:2602ace2…`, uid 999 | `valkey-cli ping` с паролем | `valkeydata:/data` (AOF) | `backend` | secret `valkey.conf` (`requirepass`) | | bootstrap | `migrate` | образ api, `restart: "no"` | код выхода 0 | нет | `backend` | secret `database_url` | +| генератор нагрузки (профиль `bench`) | `k6` | `grafana/k6:2.3.0@sha256:2dd4756e…`, uid хоста | нет | нет (скрипт `infra/k6` только для чтения) | только `edge`, порт не публикуется | нет | Готовность реплики зависит только от PostgreSQL: Valkey — кэш, при его отказе проверка идёт в БД. Общее хранилище сессий — PostgreSQL (источник истины) плюс общий Valkey. Bootstrap: `make secrets` (идемпотентен, не перезаписывает существующие файлы), `make up` (миграции — `migrate` под advisory lock); повтор безопасен. -Лимиты: lb 1 CPU/128m, api 1 CPU/512m на реплику, migrate 0.5/256m, postgres 2/1g, valkey 0.5/128m. +Лимиты: lb 1 CPU/128m, api 1 CPU/512m на реплику, migrate 0.5/256m, postgres 2/1g, valkey 0.5/128m, +k6 2/512m (`make limits` сверяет их и изоляцию с политикой). **Соответствие `.env.example` реальной конфигурации:** @@ -160,7 +218,9 @@ message_id, created_at)`, PK `(chat_id, user_id, key)` — область «по | — | `LB_HTTP_PORT`, `LB_STATS_PORT` | опубликованные порты LB на `127.0.0.1` | | — | `API_CPUS`, `API_MEMORY`, `WEB_CONCURRENCY`, `DB_POOL_MAX_SIZE`, `API_SERVER_MAXCONN` | размер реплики, воркеры, пул на воркер, очередь LB на сервер | | — | `PG_MAX_CONNECTIONS`, `PG_CPUS`, `PG_MEMORY`, `LB_CPUS` | лимиты PostgreSQL и LB | -| — | `REQUEST_TIMEOUT_SECONDS`, `SHUTDOWN_TIMEOUT_SECONDS` | 8 с и 5 с (заданы в compose и образе) | +| — | `REQUEST_TIMEOUT_SECONDS`, `DRAIN_SECONDS`, `SHUTDOWN_TIMEOUT_SECONDS` | 8 с, 2 с и 5 с (заданы в compose и образе) | +| — | `DB_ACQUIRE_TIMEOUT_SECONDS`, `DB_COMMAND_TIMEOUT_SECONDS` | 1 с и 2 с: ожидание соединения из пула и `statement_timeout` = `lock_timeout` | +| — | `K6_CPUS`, `K6_MEMORY` | лимиты генератора нагрузки (профиль `bench`) | | — | `CURSOR_SECRET_FILE` | общий ключ подписи курсора для всех реплик | ## Доказательства критериев @@ -169,39 +229,61 @@ message_id, created_at)`, PK `(chat_id, user_id, key)` — область «по | --- | --- | --- | --- | | Публичный smoke | `make test BASE_URL=http://localhost:18083` | `PASS: lab3 public API scenario` (включая replay == first, `409` на другой body, `400` без ключа, `X-Instance-Id` на каждом ответе) | `evidence/smoke.txt` | | Запуск из чистого clone | `git clone`, `make secrets`, `config --quiet`, `make up`, `make test` | `valid`; все сервисы `Healthy`, `migrate` `Exited` (0) за 15 с; smoke `PASS` | `evidence/clean-clone.txt` | -| **3.1** Весь стенд в `compose.yaml`, ≥2 API, одна точка входа, общая БД и общее хранилище сессий | `make stack-evidence` (`config --quiet`, `ps`, `port api 8080`, `docker inspect`, статистика LB) | 2 контейнера `api` без портов (`no port 8080/tcp`), порты только у `lb` на `127.0.0.1`; `container_name` 0; LB видит `api1 UP L7OK 200`, `api2 UP L7OK 200`; один `postgres`, один `valkey` | `evidence/stack.txt` | -| **3.2** Разные запросы одного пользователя — разные реплики без повторного входа, видно по `X-Instance-Id` | `make lb-check` (40 × `GET /users/me` одним токеном, новые соединения) + `docker inspect` | `{"302cfee8f673": 20, "b582bd503bdd": 20}` — ровно hostname двух контейнеров `api` | `evidence/lb-check.json`, `evidence/stack.txt` | -| **3.3** Остановка любой одной реплики: ≤10 с, данные доступны, ошибки измерены | `make failover-evidence` → `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, `docker inspect` до/после, `restore`; второй прогон после сдвига round robin | реплика `302cfee8f673`: восстановление записи 1.21 с, 123 записи после, 0 ошибок; реплика `b582bd503bdd`: 1.73 с, 118, 0; именно target в `exited`, в логе — штатная остановка gunicorn; `PASS: failover …` | `evidence/failover-302cfee8f673.txt`, `evidence/failover-b582bd503bdd.txt` | -| **3.3** Хук останавливает только заданную реплику | `scripts/stop-one.sh` с hostname, с неизвестным ID и без него | хук 0.68 с, код 0, target `exited` с кодом 0; неизвестный ID → код 3, всё работает; без переменной → код 1 | `evidence/hooks.txt` | -| **3.4** Повтор с тем же ключом не создаёт дубль; конкурентный повтор и смена реплики | `make lb-check` (8 параллельных POST с одним ключом через LB); `make student-test -k idempotency`; failover-тест (после ошибки — тот же ключ и текст) | 8 × `201` с одинаковым телом от обеих реплик (4 + 4), в истории 1 копия, другой body → `409`; in-process: 8 повторов через два экземпляра приложения → одна строка ключа | `evidence/lb-check.json`, `evidence/student-tests.txt`, `evidence/failover-*.txt` | +| **3.1** Весь стенд в `compose.yaml`, ≥2 API, одна точка входа, общая БД и общее хранилище сессий | `make stack-evidence` (`config --quiet`, `ps`, `docker port` реплик, `docker inspect`, статистика LB) | 2 контейнера `api` без портов (`docker port` пуст), порты только у `lb` на `127.0.0.1`; `container_name` 0; LB видит `api1 UP L7OK 200`, `api2 UP L7OK 200`; один `postgres`, один `valkey` | `evidence/stack.txt` | +| **3.2** Разные запросы одного пользователя — разные реплики без повторного входа, видно по `X-Instance-Id` | `make lb-check` (40 × `GET /users/me` одним токеном, новые соединения) + `docker inspect` | `{"27de23496b0f": 20, "a3536e394b3f": 20}` — ровно hostname двух контейнеров `api` | `evidence/lb-check.json`, `evidence/stack.txt` | +| **3.3** Остановка любой одной реплики: ≤10 с, данные доступны, ошибки измерены | `make failover-evidence` → `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, `docker inspect` до/после, `restore`; второй прогон после сдвига round robin | реплика `27de23496b0f`: первая запись после хука через 2.64 с от его старта, 133 записи после, 0 ошибок; реплика `a3536e394b3f`: 2.69 с, 130, 0; именно target в `exited` с кодом 0, в его логе — сигнал, окно вывода из ротации с readiness `503`, закрытие и выход; `PASS: failover …` | `evidence/failover-27de23496b0f.txt`, `evidence/failover-a3536e394b3f.txt` | +| **3.3** Хук останавливает только заданную реплику | `scripts/stop-one.sh` с hostname, с неизвестным ID и без него | хук 2.72 с (окно вывода 2 с + остановка), код 0, target `exited` с кодом 0; неизвестный ID → код 3, всё работает; без переменной → код 1 | `evidence/hooks.txt` | +| **3.4** Повтор с тем же ключом не создаёт дубль; конкурентный повтор и смена реплики | `make lb-check` (20 раундов × 8 параллельных POST с одним ключом через LB); `make student-test -k idempotency`; failover-тест (после ошибки — тот же ключ и текст) | 160 × `201` от обеих реплик (80 + 80); в каждом из 20 раундов тела равны, в истории 1 копия, другой body → `409`; in-process: 8 повторов через два экземпляра приложения → одна строка ключа | `evidence/lb-check.json`, `evidence/student-tests.txt`, `evidence/failover-*.txt` | | **3.4** Неподтверждённую запись можно безопасно повторить | `make student-test -k "rejected_attempt or expired or concurrent"`; failover-тест сверяет историю байт-в-байт | `4xx` не занимает ключ; истёкший ключ начинает заново; в истории после отказа нет дублей `id`/`text` | `evidence/student-tests.txt`, `evidence/failover-*.txt` | | **3.5** `make test`, persistence и failover проходят; ≥2 наблюдаемых реплики | `make test`, `make persistence-evidence`, `make test-failover …` | smoke `PASS`; persistence `PASS`, `StartedAt` у `lb`, обеих `api`, `postgres`, `valkey`, `migrate` после теста новее, контейнеры те же; failover `observed_instances_before: 2` | `evidence/smoke.txt`, `evidence/persistence.txt`, `evidence/failover-*.txt` | | Перенос lab2: схема PK/FK/UNIQUE и миграции с нуля, повтор без дублей | `make reset-data && make up`; `make storage-evidence` | `0001_init`, `0002_idempotency_keys` применены; повтор `up to date`, счётчики строк равны (сообщений столько же, сколько ключей) | `evidence/schema.txt`, `evidence/migrations.txt` | | Перенос lab2: Basic только на `/auth/sessions`, Bearer, logout, `X-User-Id` игнорируется | `make test`; `make student-test`; `make test-faults` | smoke `PASS`; logout `204` → `401` → `401` через LB | `evidence/smoke.txt`, `evidence/student-tests.txt`, `evidence/faults.txt` | | Перенос lab2: password KDF | `make storage-evidence` | `$argon2id$v=19$m=19456,t=2,p=1$` у всех пользователей; в `sessions` только 32-байтовые SHA-256 | `evidence/kdf.txt` | | Перенос lab2: секреты и volumes вне Git | `git ls-files secrets .env`; `git check-ignore -v …` | пусто; пути игнорируются | `evidence/secrets.txt` | -| Перенос lab2: отказы Valkey и PostgreSQL через LB | `make test-faults BASE_URL=http://localhost:18083` | 10 passed: гонка регистрации 201×1/409×49, rollback чата, остановка Valkey и PostgreSQL (`503` ≤ 0.02 с, готовность через 0.3 с), рестарт API | `evidence/faults.txt` | +| Перенос lab2: отказы Valkey и PostgreSQL через LB | `make test-faults BASE_URL=http://localhost:18083` | 10 passed: гонка регистрации 201×1/409×49, rollback чата, остановка Valkey и PostgreSQL (`503` ≤ 0.02 с; live внутри реплики `200`; LB исключил обе реплики через 0.61 с и сам отвечает `503`; готовность через 2.1 с), рестарт API | `evidence/faults.txt` | | `X-Instance-Id` на каждом ответе приложения | `make student-test -k instance` | `200`, `204`, `400`, `401`, `404`, скрытый `405`, `413`, `500` несут один ID реплики и `X-Request-Id` | `evidence/student-tests.txt` | | Непривилегированные процессы | `make stack-evidence` | api — uid 10001, lb — 99, postgres — 70, valkey — 999 | `evidence/environment.txt` | -Строки уровня «4» и трека «5» добавляются следующими openspec-изменениями лабы. +**Уровень «4».** Нагрузка — k6 open model из сети `edge` в `lb:8080`: 100 запросов/с постоянной +интенсивности (80 % `POST` сообщения с уникальным ключом, 20 % чтения первой страницы истории), 20 +пользователей и 10 чатов, прогрев 10 с, замер 60–90 с; параллельно зонд шлёт сообщение каждые 0,1 с и после +ошибки повторяет тот же ключ, а опрос статистики LB 10 раз в секунду пишет число `UP`. Счётчики LB — +дельта за прогон по backend `api`: `wretr` (повторы), `wredis` (перенаправления), `econ`, `eresp`, `hrsp_5xx`. + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| **4.1** Health/readiness, таймауты и прекращение приёма при `SIGTERM` согласованы | `make student-test` (`test_timeouts.py`, `test_drain.py`) | цепочка: LB server 10 с > запрос 8 с > БД 3 с; readiness 0,8 с < check 1 с; окно 2 с > обнаружение 1,5 с; остановка 5 с < хук 6 с < `stop_grace_period` 8 с; настоящий gunicorn по `SIGTERM`: ready/live `(503, 200)` в окне, порт закрыт через 1.69 с, запрос, принятый до сигнала и дописанный после закрытия порта, получил `201`, выход с кодом 0 через 2.11 с | `evidence/student-tests.txt`, `compose.yaml`, `infra/haproxy/haproxy.cfg`, `src/messenger/gunicorn_conf.py` | +| **4.1** LB исключает недоступный backend; период проверок и максимум обнаружения | `make detection` | период 1 с (0,5 с после провала), `fall 2`/`rise 2`; заявлено ≤1,5 с (`503`/отказ соединения) и ≤3,5 с (зависание); измерено: `SIGTERM` 0.71 с — раньше выхода процесса (2.49 с), `kill` 0.67 с, `pause` 2.76 с, PostgreSQL 1.23 с до 0 `UP` и `503 unavailable` от LB; возврат 2.0–3.1 с; все `pass: true` | `evidence/detection/{sigterm,kill,pause,postgres}.json` | +| **4.2** Последовательно остановить каждую реплику (с восстановлением предыдущей); RPS, ошибки, время восстановления | `make failover-each` (на реплику: `stop-one.sh`, 10 с без неё, `restore.sh` — всё внутри окна нагрузки) | `a3536e394b3f`: 100.02 rps, 6001 запрос k6, ошибок 0, p50/p95/p99 2.4/3.41/5.1 мс, обнаружение 1.52 с, возврат в ротацию через 14.11 с от начала действия (хук ≈2,5 с + 10 с + restore), зонд 635 записей / 0 ошибок, пока реплика исключена, её `X-Instance-Id` не встречался; `27de23496b0f`: 100.02 rps, 0 ошибок, 2.44/3.42/5.08 мс, 0.89 с, 14.39 с, 635/0; счётчики LB `wretr`/`wredis`/`econ`/`eresp`/`hrsp_5xx` — 0 | `evidence/failover-each/summary.json`, `evidence/failover-each//` | +| **4.2** Общая идемпотентность — параллельными запросами с одним ключом | `make lb-check` | 20 раундов × 8: 160 × `201` от двух реплик, 20/20 раундов с одинаковыми телами, одной копией и `409` на другой body | `evidence/lb-check.json` | +| **4.3** CPU/RAM-лимиты, API без портов в обход LB, секреты и хранилища только где нужно, нет `container_name` | `make limits` (`docker compose --profile bench config` + `docker inspect` против политики в `scripts/limits.py`) | лимиты у всех 6 сервисов (api 1/512MiB, lb 1/128MiB, postgres 2/1024MiB, valkey 0.5/128MiB, migrate 0.5/256MiB, k6 2/512MiB) и применены у запущенных; порты только у `lb` на `127.0.0.1`; секреты: api — 3, migrate — `database_url`, postgres — пароль, valkey — конфиг, lb и k6 — нет; volumes `pgdata`/`valkeydata` только у своих хранилищ; `backend` internal, k6 только в `edge`; api `read_only`; `PASS`. Подмешанные нарушения (порт и лишний секрет у api, `container_name`, снятый лимит) → 5 нарушений, код 1 | `evidence/limits.txt`, `evidence/limits-violations.txt` | +| **4.4** Rolling update под нагрузкой | `make rolling-update FROM_REV=8d28d45` (стенд на образе уровня «3», затем замена на текущую версию с запасом +1) | `lb_up_min` 2 из 2, k6 100 rps, 9000 запросов, ошибок 0, зонд 897/0; после замены отвечают только новые контейнеры (`7a4c320f9b13`, `00548d9b229e` → `28af1eaa77c8`, `f3f2f7801b56`; образ `cbd83541c3e5` → `13e52dc26774`); старые реплики без окна дали `wredis` 54, `eresp` 2 — запросы, повторённые LB на соседа | `evidence/rolling-from-8d28d45/result.json`, `action.txt` | +| **4.4** То же на текущей версии | `make rolling-update` | `lb_up_min` 2, 9001 запрос, ошибок 0, зонд 904/0, все реплики новые; все счётчики LB 0. Слои образа те же, ID отличается только provenance-аттестацией BuildKit — это эквивалентная поочерёдная замена | `evidence/rolling/result.json`, `action.txt` | +| **4.4** Миграции однократно и совместимы с работающей версией | `scripts/rolling-update.sh`; `make storage-evidence` | `migrate` запускается один раз до замены первой реплики (`migration.completed up to date`), реплики схему не трогают, `migrate` под `pg_advisory_lock`; между уровнями «3» и «4» схема не менялась; `0002_idempotency_keys` только добавляет таблицу, код лабы 2 её не замечает | `evidence/rolling*/action.txt`, `evidence/migrations.txt` | ## Эксперименты и отказы Стенд: 2 реплики × (1 CPU, 512 MiB, 2 воркера), PostgreSQL 2 CPU / 1 GiB, HAProxy 1 CPU / 128 MiB, Valkey -0.5 CPU / 128 MiB; Docker Desktop на Apple M4 Max. Нагрузка в failover-тесте — один клиент, `POST` каждые -0.1 с в течение 15 с, таймаут 2 с, повтор после ошибки с тем же ключом. +0.5 CPU / 128 MiB, k6 2 CPU / 512 MiB; Docker Desktop на Apple M4 Max. Нагрузка в failover-тесте — один клиент, +`POST` каждые 0.1 с в течение 15 с, таймаут 2 с, повтор после ошибки с тем же ключом; в строках уровня «4» — +k6 100 rps плюс зонд (см. выше). | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| stop api#1 (`stop-one.sh`, `302cfee8f673`) | запись продолжается ≤10 с, target больше не отвечает | 0 неудачных попыток, 123 записи после хука, последние 5 ответов — от `b582bd503bdd` | первая запись через 1.21 с от старта хука (хук ≈0.7 с); потерь и дублей нет | -| stop api#2 (после `restore`, `b582bd503bdd`) | то же для второй реплики | 0 неудачных попыток, 118 записей после хука | 1.73 с; потерь и дублей нет | +| stop api#1 (`stop-one.sh`, `27de23496b0f`) | запись продолжается ≤10 с, target больше не отвечает | 0 неудачных попыток, 133 записи после хука, последние 5 ответов — от `a3536e394b3f` | первая запись после хука через 2.64 с от его старта (хук ≈2,7 с из-за окна вывода; запись не прерывалась); потерь и дублей нет | +| stop api#2 (после `restore`, `a3536e394b3f`) | то же для второй реплики | 0 неудачных попыток, 130 записей после хука | 2.69 с; потерь и дублей нет | | Полный restart (`restart.sh`: lb, api ×2, postgres, valkey) | прежний токен работает, история побайтно та же | persistence `PASS`, `StartedAt` всех контейнеров обновился, ID контейнеров прежние | стенд `healthy` ≈12 с; потерь нет | -| Конкурентный одинаковый ключ (8 POST через LB) | одно сообщение, все ответы равны | 8 × `201`, 4 + 4 по репликам, 1 копия, другой body → `409` | — | +| Конкурентный одинаковый ключ (20 раундов × 8 POST через LB) | в каждом раунде одно сообщение, все ответы равны | 160 × `201`, 80 + 80 по репликам, 20/20 раундов с 1 копией, другой body → `409` | — | | Все реплики остановлены | LB отвечает `503` в формате контракта | `503 {"error":{"code":"unavailable",…,"request_id":"no-replica-1"}}` | после `make restore` — `200` | -| `docker compose stop postgres` | реплики выходят из ротации, ответы `503`, а не `500` | `503` за ≤0.02 с на ready, me, login, register; live `200` | после `start` готовность через 0.3 с без рестарта API | +| `docker compose stop postgres` | реплики выходят из ротации, ответы `503`, а не `500` | `503` за ≤0.02 с на ready, me, login, register; live внутри реплики `200`; LB исключил обе реплики за 0.61–1.23 с и отвечает `503` сам | после `start` готовность через 2.1–2.5 с без рестарта API | | `docker compose stop valkey` | проверка сессий через PostgreSQL, без анонимного доступа | действующий токен `200`, неизвестный `401`, выход `503` | после `start` выход `204` | | Повторный `migrate` | схема и данные не меняются | `up to date`, счётчики равны | — | +| `SIGTERM` реплике (`docker stop -t 6`) | LB исключает её раньше, чем она закроет порт | исключена через 0.71 с, процесс вышел через 2.49 с с кодом 0; в логе — окно 2 с, readiness `503` | возврат после `restore` 1.96 с | +| Процесс реплики убит (`docker kill`) | исключение ≤1,5 с | 0.67 с | `restore` 3.08 с | +| Процесс реплики завис (`docker pause`) | исключение ≤3,5 с по таймауту проверки | 2.76 с; через LB — `200` от соседа | после `unpause` 2.38 с | +| Поочерёдная остановка каждой реплики под 100 rps | 0 ошибок, запись не прерывается, реплика возвращается | обе реплики: k6 0 ошибок из 6001, зонд 0 из 635, p99 ≈5 мс, повторов LB 0 | обнаружение 0.89–1.52 с; снова в ротации через 14.1–14.4 с от начала действия (хук ≈2,5 с + 10 с без реплики + `restore`) | +| Rolling update с уровня «3» под 100 rps | ёмкость не ниже 2, 0 ошибок, после — только новые реплики | `lb_up_min` 2, 0 ошибок из 9000, зонд 0 из 897; старые реплики без окна: `wredis` 54, `eresp` 2 | замена двух реплик за 7.7 с | +| Rolling update текущей версии под 100 rps | то же, и без повторов LB | `lb_up_min` 2, 0 ошибок из 9001, зонд 0 из 904, все счётчики LB 0 | 11.8 с | ## Ограничения и следующие шаги @@ -213,5 +295,12 @@ message_id, created_at)`, PK `(chat_id, user_id, key)` — область «по - Таблица ключей растёт вместе с историей (строка на сообщение); истёкшие ключи только перезаписываются при повторном использовании, фоновой очистки нет. - Запись в один горячий чат сериализуется блокировкой строки чата; повтор с ключом берёт ту же блокировку. -- Время обнаружения отказа, поочерёдная остановка под нагрузкой, rolling update и замер лимитов — уровень «4»; - серии 1/2/4 реплик и поиск насыщения — трек «5». +- Окно вывода из ротации удлиняет каждую остановку реплики на 2 с; оно рассчитано на проверки LB раз в + секунду: при большем `inter` окно нужно увеличить (это ловит `test_timeouts.py`). +- Время обнаружения измерено одним прогоном на сценарий; верхняя граница — формула по параметрам проверки, а + замер её подтверждает. Время считается от запуска команды отказа, с допуском 0,25 с на docker CLI и опрос. +- Rolling update проверен со сменой кода без смены схемы. Миграция, несовместимая с работающей версией + (удаление или переименование столбца), потребовала бы двух выпусков expand/contract; в лабе таких нет. +- `scripts/rolling-update.sh` и `restore.sh` используют `--no-recreate`: после замены с другим `API_IMAGE` + обычный `make up` пересоздаст реплики под текущий конфиг — это ожидаемо. +- Серии 1/2/4 реплик и поиск насыщения — трек «5», следующее изменение. diff --git a/evidence/detection/kill.json b/evidence/detection/kill.json new file mode 100644 index 00000000..af76c3ac --- /dev/null +++ b/evidence/detection/kill.json @@ -0,0 +1,30 @@ +{ + "action": [ + "docker", + "kill", + "a3536e394b3fd136116fb8d551e1a4eedeafc169b9a2dbb59858c705539aef16" + ], + "action_exit": 0, + "action_seconds": 0.116, + "lb_up_before": 2, + "lb_up_counted_as_detected": 1, + "detection_seconds": 0.665, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 200, + "x_instance_id_present": true, + "error_code": null + }, + "restore": "scripts/restore.sh", + "restore_seconds": 3.078, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "detected_in_time": true, + "lb_answer_ok": true, + "restored": true + }, + "pass": true +} diff --git a/evidence/detection/pause.json b/evidence/detection/pause.json new file mode 100644 index 00000000..63d50ce9 --- /dev/null +++ b/evidence/detection/pause.json @@ -0,0 +1,30 @@ +{ + "action": [ + "docker", + "pause", + "a3536e394b3fd136116fb8d551e1a4eedeafc169b9a2dbb59858c705539aef16" + ], + "action_exit": 0, + "action_seconds": 0.061, + "lb_up_before": 2, + "lb_up_counted_as_detected": 1, + "detection_seconds": 2.756, + "declared_max_seconds": 3.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 200, + "x_instance_id_present": true, + "error_code": null + }, + "restore": "docker unpause a3536e394b3fd136116fb8d551e1a4eedeafc169b9a2dbb59858c705539aef16 && scripts/lb-wait.sh", + "restore_seconds": 2.383, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "detected_in_time": true, + "lb_answer_ok": true, + "restored": true + }, + "pass": true +} diff --git a/evidence/detection/postgres.json b/evidence/detection/postgres.json new file mode 100644 index 00000000..89481146 --- /dev/null +++ b/evidence/detection/postgres.json @@ -0,0 +1,30 @@ +{ + "action": [ + "docker", + "stop", + "67fe9b43b70bb4a2765caa24caadd86a1c735fef1db43690c100d0851bcb79e9" + ], + "action_exit": 0, + "action_seconds": 0.171, + "lb_up_before": 2, + "lb_up_counted_as_detected": 0, + "detection_seconds": 1.227, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 503, + "x_instance_id_present": false, + "error_code": "unavailable" + }, + "restore": "docker start 67fe9b43b70bb4a2765caa24caadd86a1c735fef1db43690c100d0851bcb79e9 && scripts/lb-wait.sh", + "restore_seconds": 2.499, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "detected_in_time": true, + "lb_answer_ok": true, + "restored": true + }, + "pass": true +} diff --git a/evidence/detection/sigterm.json b/evidence/detection/sigterm.json new file mode 100644 index 00000000..8ca53839 --- /dev/null +++ b/evidence/detection/sigterm.json @@ -0,0 +1,33 @@ +{ + "action": [ + "docker", + "stop", + "-t", + "6", + "a3536e394b3fd136116fb8d551e1a4eedeafc169b9a2dbb59858c705539aef16" + ], + "action_exit": 0, + "action_seconds": 2.493, + "lb_up_before": 2, + "lb_up_counted_as_detected": 1, + "detection_seconds": 0.706, + "declared_max_seconds": 1.5, + "measurement_slack_seconds": 0.25, + "lb_answer_after_detection": { + "path": "/health/live", + "status": 200, + "x_instance_id_present": true, + "error_code": null + }, + "restore": "scripts/restore.sh", + "restore_seconds": 1.959, + "lb_up_after_restore": 2, + "checks": { + "action_ok": true, + "detected_in_time": true, + "lb_answer_ok": true, + "excluded_before_action_end": true, + "restored": true + }, + "pass": true +} diff --git a/evidence/environment.txt b/evidence/environment.txt index b2c1dc8c..62956a20 100644 --- a/evidence/environment.txt +++ b/evidence/environment.txt @@ -1,4 +1,4 @@ -date: 2026-10-28T08:48:40Z commit: 0796e29 +date: 2026-10-28T16:53:56Z commit: e00eb6c host: Apple M4 Max, 16 CPU, 48 GiB RAM, macOS 15.1 docker: 28.3.2, VM: 16 CPU, 8217190400 bytes compose: 2.38.2-desktop.1 @@ -7,6 +7,7 @@ images: image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b image: postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b image: valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a + image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8 FROM python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv FROM python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime diff --git a/evidence/failover-27de23496b0f.txt b/evidence/failover-27de23496b0f.txt new file mode 100644 index 00000000..6cc8b6c1 --- /dev/null +++ b/evidence/failover-27de23496b0f.txt @@ -0,0 +1,38 @@ +date: 2026-10-28T16:55:49Z commit: e00eb6c +== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt +a3536e394b3f running started=2026-10-28T16:55:38.101692511Z finished=2026-10-28T16:55:25.134036046Z +27de23496b0f running started=2026-10-28T16:55:20.350443794Z finished=2026-10-28T16:55:07.575185552Z +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18083" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 2.638} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +== after: api replicas +a3536e394b3f running started=2026-10-28T16:55:38.101692511Z finished=2026-10-28T16:55:25.134036046Z +27de23496b0f exited started=2026-10-28T16:55:20.350443794Z finished=2026-10-28T16:55:52.113620836Z +== stopped replica: 27de23496b0f; shutdown: signal, drain window with readiness 503, close, exit +{"event":"Handling signal: term","level":"info","ts":"2026-10-28T16:55:49.715606Z"} +{"event":"Draining for 2.0 s","level":"info","ts":"2026-10-28T16:55:49.748209Z"} +{"event":"Draining for 2.0 s","level":"info","ts":"2026-10-28T16:55:49.800251Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"a5298514a38548218bda2d37c01dada5","duration_ms":0.391,"event":"request.handled","level":"info","ts":"2026-10-28T16:55:49.826086Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"5547700c46e843b79f7d7ff06c7a5f2e","duration_ms":0.357,"event":"request.handled","level":"info","ts":"2026-10-28T16:55:50.328563Z"} +{"event":"Shutting down","level":"info","ts":"2026-10-28T16:55:51.748865Z"} +{"event":"Shutting down","level":"info","ts":"2026-10-28T16:55:51.801844Z"} +{"event":"service.stopped","level":"info","ts":"2026-10-28T16:55:51.851863Z"} +{"event":"Finished server process [7]","level":"info","ts":"2026-10-28T16:55:51.852006Z"} +{"event":"service.stopped","level":"info","ts":"2026-10-28T16:55:51.904178Z"} +{"event":"Finished server process [8]","level":"info","ts":"2026-10-28T16:55:51.904314Z"} +{"event":"Worker (pid:7) was sent SIGTERM!","level":"info","ts":"2026-10-28T16:55:51.951844Z"} +{"event":"Worker (pid:8) was sent SIGTERM!","level":"info","ts":"2026-10-28T16:55:51.951924Z"} +{"event":"Shutting down: Master","level":"info","ts":"2026-10-28T16:55:52.052865Z"} +$ scripts/restore.sh + Container mtusi2026lab3-api-1 Running + Container mtusi2026lab3-api-2 Starting + Container mtusi2026lab3-api-2 Started + Container mtusi2026lab3-api-2 Waiting + Container mtusi2026lab3-api-1 Waiting + Container mtusi2026lab3-api-2 Healthy + Container mtusi2026lab3-api-1 Healthy +== after restore +a3536e394b3f running started=2026-10-28T16:55:38.101692511Z finished=2026-10-28T16:55:25.134036046Z +27de23496b0f running started=2026-10-28T16:56:05.100393509Z finished=2026-10-28T16:55:52.113620836Z diff --git a/evidence/failover-302cfee8f673.txt b/evidence/failover-302cfee8f673.txt deleted file mode 100644 index e03e34a8..00000000 --- a/evidence/failover-302cfee8f673.txt +++ /dev/null @@ -1,30 +0,0 @@ -date: 2026-10-28T08:47:27Z commit: 0796e29 -== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt -302cfee8f673 running started=2026-10-28T08:47:14.781792917Z finished=2026-10-28T08:47:07.856233678Z -b582bd503bdd running started=2026-10-28T08:47:14.713654417Z finished=2026-10-28T08:47:07.971063344Z -$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh -python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18083" -Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 1.212} -PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. -== after: api replicas -302cfee8f673 exited started=2026-10-28T08:47:14.781792917Z finished=2026-10-28T08:47:28.65076559Z -b582bd503bdd running started=2026-10-28T08:47:14.713654417Z finished=2026-10-28T08:47:07.971063344Z -== stopped replica: 302cfee8f673; last log lines of its process -{"event":"service.stopped","level":"info","ts":"2026-10-28T08:47:28.456077Z"} -{"event":"Application shutdown complete.","level":"info","ts":"2026-10-28T08:47:28.456276Z"} -{"event":"Finished server process [8]","level":"info","ts":"2026-10-28T08:47:28.456336Z"} -{"event":"Worker (pid:7) was sent SIGTERM!","level":"info","ts":"2026-10-28T08:47:28.459598Z"} -{"event":"Worker (pid:8) was sent SIGTERM!","level":"info","ts":"2026-10-28T08:47:28.459755Z"} -{"event":"Shutting down: Master","level":"info","ts":"2026-10-28T08:47:28.568492Z"} -$ scripts/restore.sh - Container mtusi2026lab3-api-2 Running - Container mtusi2026lab3-api-1 Starting - Container mtusi2026lab3-api-1 Started - Container mtusi2026lab3-api-2 Waiting - Container mtusi2026lab3-api-1 Waiting - Container mtusi2026lab3-api-2 Healthy - Container mtusi2026lab3-api-1 Healthy -== after restore -302cfee8f673 running started=2026-10-28T08:47:43.609172555Z finished=2026-10-28T08:47:28.65076559Z -b582bd503bdd running started=2026-10-28T08:47:14.713654417Z finished=2026-10-28T08:47:07.971063344Z diff --git a/evidence/failover-a3536e394b3f.txt b/evidence/failover-a3536e394b3f.txt new file mode 100644 index 00000000..9a0e2f12 --- /dev/null +++ b/evidence/failover-a3536e394b3f.txt @@ -0,0 +1,38 @@ +date: 2026-10-28T16:56:07Z commit: e00eb6c +== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt +a3536e394b3f running started=2026-10-28T16:55:38.101692511Z finished=2026-10-28T16:55:25.134036046Z +27de23496b0f running started=2026-10-28T16:56:05.100393509Z finished=2026-10-28T16:55:52.113620836Z +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18083" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 2.689} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +== after: api replicas +a3536e394b3f exited started=2026-10-28T16:55:38.101692511Z finished=2026-10-28T16:56:10.104800095Z +27de23496b0f running started=2026-10-28T16:56:05.100393509Z finished=2026-10-28T16:55:52.113620836Z +== stopped replica: a3536e394b3f; shutdown: signal, drain window with readiness 503, close, exit +{"event":"Handling signal: term","level":"info","ts":"2026-10-28T16:56:07.660163Z"} +{"event":"Draining for 2.0 s","level":"info","ts":"2026-10-28T16:56:07.716933Z"} +{"event":"Draining for 2.0 s","level":"info","ts":"2026-10-28T16:56:07.755310Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"9015463a87b2431bbba6f0a3804b81ef","duration_ms":1.467,"event":"request.handled","level":"info","ts":"2026-10-28T16:56:08.155658Z"} +{"method":"GET","route":"/health/ready","status":503,"request_id":"ca8d0e81fc814140a89b461975ec2c0c","duration_ms":0.441,"event":"request.handled","level":"info","ts":"2026-10-28T16:56:08.658832Z"} +{"event":"Shutting down","level":"info","ts":"2026-10-28T16:56:09.722798Z"} +{"event":"Shutting down","level":"info","ts":"2026-10-28T16:56:09.757396Z"} +{"event":"service.stopped","level":"info","ts":"2026-10-28T16:56:09.838472Z"} +{"event":"Finished server process [7]","level":"info","ts":"2026-10-28T16:56:09.838854Z"} +{"event":"service.stopped","level":"info","ts":"2026-10-28T16:56:09.862674Z"} +{"event":"Finished server process [8]","level":"info","ts":"2026-10-28T16:56:09.863018Z"} +{"event":"Worker (pid:7) was sent SIGTERM!","level":"info","ts":"2026-10-28T16:56:09.932410Z"} +{"event":"Worker (pid:8) was sent SIGTERM!","level":"info","ts":"2026-10-28T16:56:09.932568Z"} +{"event":"Shutting down: Master","level":"info","ts":"2026-10-28T16:56:10.033607Z"} +$ scripts/restore.sh + Container mtusi2026lab3-api-2 Running + Container mtusi2026lab3-api-1 Starting + Container mtusi2026lab3-api-1 Started + Container mtusi2026lab3-api-2 Waiting + Container mtusi2026lab3-api-1 Waiting + Container mtusi2026lab3-api-2 Healthy + Container mtusi2026lab3-api-1 Healthy +== after restore +a3536e394b3f running started=2026-10-28T16:56:23.072055754Z finished=2026-10-28T16:56:10.104800095Z +27de23496b0f running started=2026-10-28T16:56:05.100393509Z finished=2026-10-28T16:55:52.113620836Z diff --git a/evidence/failover-b582bd503bdd.txt b/evidence/failover-b582bd503bdd.txt deleted file mode 100644 index 83985cc4..00000000 --- a/evidence/failover-b582bd503bdd.txt +++ /dev/null @@ -1,30 +0,0 @@ -date: 2026-10-28T08:47:47Z commit: 0796e29 -== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt -302cfee8f673 running started=2026-10-28T08:47:43.609172555Z finished=2026-10-28T08:47:28.65076559Z -b582bd503bdd running started=2026-10-28T08:47:14.713654417Z finished=2026-10-28T08:47:07.971063344Z -$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh -python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18083" -Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 118, "failed_attempts": 0, "first_recovery_seconds": 1.726} -PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. -== after: api replicas -302cfee8f673 running started=2026-10-28T08:47:43.609172555Z finished=2026-10-28T08:47:28.65076559Z -b582bd503bdd exited started=2026-10-28T08:47:14.713654417Z finished=2026-10-28T08:47:48.23612871Z -== stopped replica: b582bd503bdd; last log lines of its process -{"event":"service.stopped","level":"info","ts":"2026-10-28T08:47:47.995678Z"} -{"event":"Application shutdown complete.","level":"info","ts":"2026-10-28T08:47:47.995877Z"} -{"event":"Finished server process [8]","level":"info","ts":"2026-10-28T08:47:47.995948Z"} -{"event":"Worker (pid:7) was sent SIGTERM!","level":"info","ts":"2026-10-28T08:47:48.041499Z"} -{"event":"Worker (pid:8) was sent SIGTERM!","level":"info","ts":"2026-10-28T08:47:48.041664Z"} -{"event":"Shutting down: Master","level":"info","ts":"2026-10-28T08:47:48.152563Z"} -$ scripts/restore.sh - Container mtusi2026lab3-api-1 Running - Container mtusi2026lab3-api-2 Starting - Container mtusi2026lab3-api-2 Started - Container mtusi2026lab3-api-2 Waiting - Container mtusi2026lab3-api-1 Waiting - Container mtusi2026lab3-api-2 Healthy - Container mtusi2026lab3-api-1 Healthy -== after restore -302cfee8f673 running started=2026-10-28T08:47:43.609172555Z finished=2026-10-28T08:47:28.65076559Z -b582bd503bdd running started=2026-10-28T08:48:03.240408759Z finished=2026-10-28T08:47:48.23612871Z diff --git a/evidence/failover-each/27de23496b0f/action.txt b/evidence/failover-each/27de23496b0f/action.txt new file mode 100644 index 00000000..c9544d21 --- /dev/null +++ b/evidence/failover-each/27de23496b0f/action.txt @@ -0,0 +1,7 @@ + Container mtusi2026lab3-api-1 Running + Container mtusi2026lab3-api-2 Starting + Container mtusi2026lab3-api-2 Started + Container mtusi2026lab3-api-2 Waiting + Container mtusi2026lab3-api-1 Waiting + Container mtusi2026lab3-api-2 Healthy + Container mtusi2026lab3-api-1 Healthy diff --git a/evidence/failover-each/27de23496b0f/k6/k6.txt b/evidence/failover-each/27de23496b0f/k6/k6.txt new file mode 100644 index 00000000..a7278f06 --- /dev/null +++ b/evidence/failover-each/27de23496b0f/k6/k6.txt @@ -0,0 +1,2 @@ +time="2026-10-28T16:59:32Z" level=info msg=LOAD-START source=console +measure: 6001 requests, failed ratio 0 diff --git a/evidence/failover-each/27de23496b0f/k6/summary.json b/evidence/failover-each/27de23496b0f/k6/summary.json new file mode 100644 index 00000000..1bddeac3 --- /dev/null +++ b/evidence/failover-each/27de23496b0f/k6/summary.json @@ -0,0 +1,309 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "message 201", + "path": "::message 201", + "id": "6445aff337e1ade0bc10402fbac79882", + "passes": 5635, + "fails": 0 + }, + { + "id": "fe27b3b8edd292c7f2170a6357b087d0", + "passes": 1366, + "fails": 0, + "name": "history 200", + "path": "::history 200" + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "p(90)", + "p(95)", + "p(99)", + "max" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": false, + "isStdErrTTY": false, + "testRunDurationMs": 70455.419797 + }, + "metrics": { + "dropped_iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "rate": 0, + "count": 0 + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 2.5045669431286224, + "min": 0.881334, + "med": 2.445916, + "p(90)": 3.14025, + "p(95)": 3.5028959999999993, + "p(99)": 5.572145999999999, + "max": 35.650709 + } + }, + "http_req_waiting": { + "values": { + "min": 0.854584, + "med": 2.39875, + "p(90)": 3.084625, + "p(95)": 3.4471459999999996, + "p(99)": 5.454729499999997, + "max": 35.612583, + "avg": 2.456700437242939 + }, + "type": "trend", + "contains": "time" + }, + "data_sent": { + "values": { + "rate": 29177.10242764789, + "count": 2055685 + }, + "type": "counter", + "contains": "data" + }, + "http_req_duration{scenario:measure,name:post_message}": { + "type": "trend", + "contains": "time", + "values": { + "min": 0.975417, + "med": 2.468, + "p(90)": 3.1078086000000003, + "p(95)": 3.387708, + "p(99)": 5.072010919999999, + "max": 18.763375, + "avg": 2.4617575886847023 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "p(90)": 0, + "p(95)": 0, + "p(99)": 0, + "max": 0 + } + }, + "http_req_duration{scenario:measure}": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 3.417125, + "p(99)": 5.077208, + "max": 35.650709, + "avg": 2.4559260606565587, + "min": 0.947959, + "med": 2.437417, + "p(90)": 3.108041 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 16740045, + "rate": 237597.69011713122 + } + }, + "http_reqs": { + "contains": "default", + "values": { + "count": 7051, + "rate": 100.07746771384977 + }, + "type": "counter" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "med": 0.001917, + "p(90)": 0.002916, + "p(95)": 0.003708, + "p(99)": 0.013229499999999991, + "max": 0.543834, + "avg": 0.002898540207062831, + "min": 0.000583 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 3.244959, + "p(95)": 3.575416, + "p(99)": 5.421083, + "max": 35.723541, + "avg": 2.564868511498362, + "min": 0.96825, + "med": 2.543542 + } + }, + "http_req_failed{scenario:measure}": { + "type": "rate", + "contains": "default", + "values": { + "passes": 0, + "fails": 6001, + "rate": 0 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 7001, + "rate": 99.36779910149798 + } + }, + "dropped_iterations": { + "values": { + "count": 0, + "rate": 0 + }, + "type": "counter", + "contains": "default" + }, + "http_req_duration{scenario:measure,name:get_history}": { + "type": "trend", + "contains": "time", + "values": { + "min": 0.947959, + "med": 2.3457084999999998, + "p(90)": 3.1207208000000004, + "p(95)": 3.5455186999999997, + "p(99)": 5.06117319, + "max": 35.650709, + "avg": 2.431537381692576 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "min": 0.008875, + "med": 0.026584, + "p(90)": 0.041625, + "p(95)": 0.052645999999999984, + "p(99)": 0.09612449999999996, + "max": 0.510625, + "avg": 0.029690495532548652 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 7001, + "fails": 0 + } + }, + "http_req_failed": { + "values": { + "rate": 0, + "passes": 0, + "fails": 7051 + }, + "type": "rate", + "contains": "default" + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 20, + "min": 20, + "max": 20 + } + }, + "http_reqs{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 6001, + "rate": 85.17442685446213 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "max": 35.650709, + "avg": 2.5045669431286224, + "min": 0.881334, + "med": 2.445916, + "p(90)": 3.14025, + "p(95)": 3.5028959999999993, + "p(99)": 5.572145999999999 + } + }, + "http_req_sending": { + "values": { + "max": 0.43, + "avg": 0.018176010353141377, + "min": 0.002458, + "med": 0.016167, + "p(90)": 0.021666, + "p(95)": 0.030874499999999975, + "p(99)": 0.075583 + }, + "type": "trend", + "contains": "time" + }, + "iterations{scenario:measure}": { + "contains": "default", + "values": { + "count": 6001, + "rate": 85.17442685446213 + }, + "type": "counter" + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "max": 0.25825, + "avg": 0.0004749090909090909, + "min": 0, + "med": 0, + "p(90)": 0, + "p(95)": 0, + "p(99)": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 0, + "min": 0, + "max": 1 + } + } + } +} \ No newline at end of file diff --git a/evidence/failover-each/27de23496b0f/lb_up.csv b/evidence/failover-each/27de23496b0f/lb_up.csv new file mode 100644 index 00000000..464072be --- /dev/null +++ b/evidence/failover-each/27de23496b0f/lb_up.csv @@ -0,0 +1,644 @@ +t_rel_s,up +-25.719,2 +-25.615,2 +-25.51,2 +-25.406,2 +-25.293,2 +-25.189,2 +-25.081,2 +-24.974,2 +-24.867,2 +-24.758,2 +-24.647,2 +-24.54,2 +-24.424,2 +-24.318,2 +-24.207,2 +-24.098,2 +-23.986,2 +-23.871,2 +-23.764,2 +-23.654,2 +-23.539,2 +-23.432,2 +-23.317,2 +-23.212,2 +-23.098,2 +-22.985,2 +-22.872,2 +-22.757,2 +-22.651,2 +-22.543,2 +-22.43,2 +-22.315,2 +-22.201,2 +-22.089,2 +-21.975,2 +-21.859,2 +-21.752,2 +-21.639,2 +-21.531,2 +-21.417,2 +-21.306,2 +-21.198,2 +-21.086,2 +-20.981,2 +-20.872,2 +-20.764,2 +-20.649,2 +-20.536,2 +-20.429,2 +-20.322,2 +-20.212,2 +-20.095,2 +-19.983,2 +-19.876,2 +-19.768,2 +-19.653,2 +-19.542,2 +-19.427,2 +-19.318,2 +-19.21,2 +-19.098,2 +-18.986,2 +-18.875,2 +-18.77,2 +-18.655,2 +-18.54,2 +-18.431,2 +-18.315,2 +-18.204,2 +-18.094,2 +-17.981,2 +-17.873,2 +-17.764,2 +-17.649,2 +-17.535,2 +-17.429,2 +-17.314,2 +-17.207,2 +-17.092,2 +-16.979,2 +-16.867,2 +-16.762,2 +-16.647,2 +-16.539,2 +-16.434,2 +-16.32,2 +-16.206,2 +-16.095,2 +-15.987,2 +-15.883,2 +-15.777,2 +-15.673,2 +-15.563,2 +-15.452,2 +-15.337,2 +-15.227,2 +-15.123,2 +-15.006,2 +-14.891,2 +-14.776,2 +-14.662,2 +-14.549,2 +-14.436,2 +-14.332,2 +-14.225,2 +-14.116,2 +-14.003,2 +-13.889,2 +-13.781,2 +-13.673,2 +-13.558,2 +-13.443,2 +-13.331,2 +-13.221,2 +-13.105,2 +-12.997,2 +-12.884,2 +-12.772,2 +-12.658,2 +-12.548,2 +-12.44,2 +-12.328,2 +-12.222,2 +-12.114,2 +-12.003,2 +-11.898,2 +-11.789,2 +-11.681,2 +-11.572,2 +-11.458,2 +-11.343,2 +-11.238,2 +-11.131,2 +-11.024,2 +-10.91,2 +-10.796,2 +-10.681,2 +-10.566,2 +-10.453,2 +-10.336,2 +-10.222,2 +-10.114,2 +-10.0,2 +-9.889,2 +-9.78,2 +-9.674,2 +-9.563,2 +-9.449,2 +-9.332,2 +-9.218,2 +-9.111,2 +-9.002,2 +-8.887,2 +-8.781,2 +-8.675,2 +-8.56,2 +-8.447,2 +-8.334,2 +-8.222,2 +-8.111,2 +-7.998,2 +-7.882,2 +-7.769,2 +-7.653,2 +-7.54,2 +-7.43,2 +-7.321,2 +-7.209,2 +-7.096,2 +-6.985,2 +-6.873,2 +-6.763,2 +-6.651,2 +-6.537,2 +-6.423,2 +-6.313,2 +-6.201,2 +-6.084,2 +-5.97,2 +-5.864,2 +-5.749,2 +-5.642,2 +-5.533,2 +-5.418,2 +-5.306,2 +-5.196,2 +-5.083,2 +-4.968,2 +-4.857,2 +-4.749,2 +-4.639,2 +-4.533,2 +-4.42,2 +-4.315,2 +-4.209,2 +-4.101,2 +-3.991,2 +-3.883,2 +-3.772,2 +-3.66,2 +-3.548,2 +-3.439,2 +-3.331,2 +-3.215,2 +-3.105,2 +-2.997,2 +-2.886,2 +-2.781,2 +-2.665,2 +-2.55,2 +-2.434,2 +-2.322,2 +-2.214,2 +-2.1,2 +-1.985,2 +-1.873,2 +-1.761,2 +-1.652,2 +-1.54,2 +-1.434,2 +-1.322,2 +-1.208,2 +-1.097,2 +-0.98,2 +-0.865,2 +-0.756,2 +-0.642,2 +-0.532,2 +-0.424,2 +-0.31,2 +-0.197,2 +-0.089,2 +0.017,2 +0.126,2 +0.238,2 +0.349,2 +0.455,2 +0.56,2 +0.671,2 +0.78,2 +0.894,1 +1.009,1 +1.123,1 +1.235,1 +1.344,1 +1.458,1 +1.564,1 +1.679,1 +1.793,1 +1.906,1 +2.015,1 +2.122,1 +2.236,1 +2.351,1 +2.462,1 +2.569,1 +2.682,1 +2.79,1 +2.9,1 +3.012,1 +3.118,1 +3.231,1 +3.344,1 +3.459,1 +3.574,1 +3.686,1 +3.792,1 +3.908,1 +4.022,1 +4.139,1 +4.255,1 +4.37,1 +4.484,1 +4.598,1 +4.714,1 +4.828,1 +4.936,1 +5.052,1 +5.162,1 +5.277,1 +5.389,1 +5.504,1 +5.616,1 +5.728,1 +5.839,1 +5.953,1 +6.059,1 +6.175,1 +6.291,1 +6.403,1 +6.514,1 +6.628,1 +6.742,1 +6.848,1 +6.958,1 +7.069,1 +7.185,1 +7.294,1 +7.399,1 +7.513,1 +7.627,1 +7.735,1 +7.848,1 +7.956,1 +8.071,1 +8.18,1 +8.285,1 +8.394,1 +8.508,1 +8.618,1 +8.732,1 +8.844,1 +8.959,1 +9.069,1 +9.177,1 +9.284,1 +9.396,1 +9.503,1 +9.616,1 +9.728,1 +9.832,1 +9.945,1 +10.053,1 +10.159,1 +10.272,1 +10.377,1 +10.491,1 +10.608,1 +10.722,1 +10.838,1 +10.948,1 +11.053,1 +11.169,1 +11.276,1 +11.391,1 +11.502,1 +11.609,1 +11.724,1 +11.834,1 +11.949,1 +12.064,1 +12.181,1 +12.295,1 +12.408,1 +12.518,1 +12.632,1 +12.739,1 +12.844,1 +12.957,1 +13.07,1 +13.179,1 +13.285,1 +13.399,1 +13.511,1 +13.617,1 +13.729,1 +13.843,1 +13.948,1 +14.062,1 +14.17,1 +14.278,1 +14.392,2 +14.497,2 +14.607,2 +14.718,2 +14.829,2 +14.945,2 +15.061,2 +15.175,2 +15.283,2 +15.391,2 +15.504,2 +15.617,2 +15.731,2 +15.844,2 +15.953,2 +16.065,2 +16.182,2 +16.296,2 +16.402,2 +16.515,2 +16.623,2 +16.73,2 +16.84,2 +16.953,2 +17.068,2 +17.183,2 +17.29,2 +17.405,2 +17.514,2 +17.628,2 +17.733,2 +17.84,2 +17.95,2 +18.067,2 +18.176,2 +18.293,2 +18.409,2 +18.52,2 +18.627,2 +18.74,2 +18.843,2 +18.953,2 +19.061,2 +19.168,2 +19.278,2 +19.386,2 +19.498,2 +19.613,2 +19.718,2 +19.827,2 +19.938,2 +20.045,2 +20.162,2 +20.278,2 +20.386,2 +20.501,2 +20.611,2 +20.719,2 +20.835,2 +20.944,2 +21.055,2 +21.17,2 +21.277,2 +21.393,2 +21.497,2 +21.612,2 +21.728,2 +21.841,2 +21.953,2 +22.062,2 +22.175,2 +22.29,2 +22.403,2 +22.509,2 +22.617,2 +22.722,2 +22.837,2 +22.951,2 +23.06,2 +23.168,2 +23.278,2 +23.386,2 +23.497,2 +23.608,2 +23.719,2 +23.827,2 +23.93,2 +24.042,2 +24.15,2 +24.261,2 +24.374,2 +24.482,2 +24.59,2 +24.699,2 +24.814,2 +24.928,2 +25.043,2 +25.159,2 +25.269,2 +25.378,2 +25.492,2 +25.602,2 +25.709,2 +25.821,2 +25.935,2 +26.049,2 +26.165,2 +26.271,2 +26.381,2 +26.496,2 +26.602,2 +26.714,2 +26.83,2 +26.936,2 +27.045,2 +27.161,2 +27.268,2 +27.382,2 +27.496,2 +27.601,2 +27.714,2 +27.83,2 +27.94,2 +28.055,2 +28.164,2 +28.27,2 +28.378,2 +28.487,2 +28.591,2 +28.698,2 +28.807,2 +28.922,2 +29.034,2 +29.144,2 +29.255,2 +29.369,2 +29.476,2 +29.585,2 +29.7,2 +29.807,2 +29.915,2 +30.028,2 +30.135,2 +30.24,2 +30.342,2 +30.448,2 +30.553,2 +30.669,2 +30.775,2 +30.89,2 +31.003,2 +31.117,2 +31.228,2 +31.344,2 +31.459,2 +31.569,2 +31.684,2 +31.794,2 +31.901,2 +32.01,2 +32.121,2 +32.228,2 +32.339,2 +32.443,2 +32.548,2 +32.656,2 +32.768,2 +32.883,2 +32.996,2 +33.111,2 +33.226,2 +33.337,2 +33.443,2 +33.55,2 +33.662,2 +33.769,2 +33.878,2 +33.99,2 +34.099,2 +34.213,2 +34.326,2 +34.436,2 +34.548,2 +34.65,2 +34.761,2 +34.872,2 +34.986,2 +35.101,2 +35.217,2 +35.331,2 +35.447,2 +35.564,2 +35.669,2 +35.778,2 +35.89,2 +36.001,2 +36.11,2 +36.219,2 +36.335,2 +36.442,2 +36.555,2 +36.67,2 +36.777,2 +36.887,2 +36.999,2 +37.109,2 +37.216,2 +37.331,2 +37.444,2 +37.555,2 +37.661,2 +37.772,2 +37.878,2 +37.987,2 +38.103,2 +38.209,2 +38.324,2 +38.436,2 +38.544,2 +38.653,2 +38.763,2 +38.878,2 +38.994,2 +39.101,2 +39.217,2 +39.324,2 +39.432,2 +39.545,2 +39.658,2 +39.773,2 +39.88,2 +39.994,2 +40.11,2 +40.216,2 +40.328,2 +40.441,2 +40.553,2 +40.659,2 +40.774,2 +40.886,2 +40.997,2 +41.112,2 +41.219,2 +41.328,2 +41.443,2 +41.558,2 +41.67,2 +41.786,2 +41.901,2 +42.015,2 +42.132,2 +42.245,2 +42.358,2 +42.468,2 +42.583,2 +42.69,2 +42.804,2 +42.915,2 +43.032,2 +43.149,2 +43.271,2 +43.388,2 +43.496,2 +43.612,2 +43.721,2 +43.838,2 +43.944,2 +44.055,2 +44.169,2 +44.281,2 +44.388,2 +44.501,2 +44.611,2 +44.725,2 +44.84,2 +44.953,2 +45.065,2 +45.173,2 +45.288,2 +45.404,2 +45.523,2 +45.629,2 diff --git a/evidence/failover-each/27de23496b0f/probe.csv b/evidence/failover-each/27de23496b0f/probe.csv new file mode 100644 index 00000000..b1a0549e --- /dev/null +++ b/evidence/failover-each/27de23496b0f/probe.csv @@ -0,0 +1,636 @@ +t_rel_s,status,latency_ms,instance +-25.719,201,6.4,a3536e394b3f +-25.605,201,5.0,27de23496b0f +-25.499,201,4.4,a3536e394b3f +-25.393,201,5.6,27de23496b0f +-25.283,201,4.8,a3536e394b3f +-25.176,201,3.7,27de23496b0f +-25.062,201,4.5,27de23496b0f +-24.955,201,4.0,27de23496b0f +-24.849,201,4.3,27de23496b0f +-24.745,201,5.0,a3536e394b3f +-24.639,201,5.7,a3536e394b3f +-24.523,201,6.9,a3536e394b3f +-24.409,201,7.9,27de23496b0f +-24.293,201,6.7,27de23496b0f +-24.186,201,7.2,27de23496b0f +-24.071,201,7.1,a3536e394b3f +-23.954,201,7.2,a3536e394b3f +-23.839,201,7.0,27de23496b0f +-23.722,201,7.7,27de23496b0f +-23.612,201,6.9,27de23496b0f +-23.497,201,7.3,a3536e394b3f +-23.38,201,8.5,27de23496b0f +-23.264,201,6.5,27de23496b0f +-23.15,201,8.0,a3536e394b3f +-23.038,201,7.6,a3536e394b3f +-22.92,201,8.0,27de23496b0f +-22.811,201,10.0,a3536e394b3f +-22.697,201,5.7,27de23496b0f +-22.585,201,3.9,27de23496b0f +-22.478,201,4.5,27de23496b0f +-22.364,201,4.2,27de23496b0f +-22.258,201,5.6,27de23496b0f +-22.142,201,5.8,27de23496b0f +-22.031,201,6.9,27de23496b0f +-21.916,201,5.6,a3536e394b3f +-21.806,201,5.9,a3536e394b3f +-21.695,201,7.6,a3536e394b3f +-21.581,201,7.7,27de23496b0f +-21.465,201,6.3,27de23496b0f +-21.348,201,7.5,a3536e394b3f +-21.239,201,5.9,a3536e394b3f +-21.132,201,6.7,27de23496b0f +-21.021,201,6.8,a3536e394b3f +-20.904,201,6.4,a3536e394b3f +-20.789,201,10.4,27de23496b0f +-20.669,201,8.2,a3536e394b3f +-20.551,201,6.8,27de23496b0f +-20.434,201,7.6,27de23496b0f +-20.323,201,10.0,a3536e394b3f +-20.211,201,8.7,a3536e394b3f +-20.092,201,7.2,a3536e394b3f +-19.978,201,6.3,27de23496b0f +-19.862,201,6.6,27de23496b0f +-19.747,201,7.9,a3536e394b3f +-19.632,201,6.9,a3536e394b3f +-19.516,201,7.2,27de23496b0f +-19.399,201,7.4,a3536e394b3f +-19.281,201,9.7,27de23496b0f +-19.169,201,5.7,27de23496b0f +-19.056,201,5.8,27de23496b0f +-18.942,201,5.9,27de23496b0f +-18.831,201,6.3,a3536e394b3f +-18.714,201,6.1,a3536e394b3f +-18.6,201,8.5,27de23496b0f +-18.486,201,7.5,27de23496b0f +-18.368,201,8.8,27de23496b0f +-18.25,201,8.6,27de23496b0f +-18.136,201,6.5,27de23496b0f +-18.023,201,9.0,27de23496b0f +-17.908,201,9.0,a3536e394b3f +-17.798,201,6.9,a3536e394b3f +-17.687,201,7.3,a3536e394b3f +-17.578,201,7.6,a3536e394b3f +-17.466,201,4.5,a3536e394b3f +-17.359,201,5.0,a3536e394b3f +-17.249,201,4.0,a3536e394b3f +-17.138,201,4.7,a3536e394b3f +-17.026,201,5.2,a3536e394b3f +-16.912,201,5.0,a3536e394b3f +-16.797,201,5.6,27de23496b0f +-16.691,201,7.4,27de23496b0f +-16.574,201,7.6,27de23496b0f +-16.46,201,8.2,a3536e394b3f +-16.342,201,6.6,a3536e394b3f +-16.226,201,8.1,27de23496b0f +-16.114,201,6.8,27de23496b0f +-15.997,201,7.1,a3536e394b3f +-15.885,201,6.9,a3536e394b3f +-15.775,201,3.3,a3536e394b3f +-15.667,201,3.6,a3536e394b3f +-15.563,201,5.5,27de23496b0f +-15.447,201,6.2,a3536e394b3f +-15.34,201,8.3,a3536e394b3f +-15.231,201,9.1,a3536e394b3f +-15.12,201,5.7,27de23496b0f +-15.007,201,7.0,a3536e394b3f +-14.89,201,5.1,a3536e394b3f +-14.78,201,6.4,a3536e394b3f +-14.673,201,6.7,a3536e394b3f +-14.558,201,8.4,27de23496b0f +-14.44,201,8.3,27de23496b0f +-14.322,201,6.2,a3536e394b3f +-14.206,201,7.5,27de23496b0f +-14.089,201,7.4,a3536e394b3f +-13.973,201,7.0,a3536e394b3f +-13.857,201,7.3,27de23496b0f +-13.739,201,6.6,27de23496b0f +-13.631,201,6.5,27de23496b0f +-13.514,201,7.7,a3536e394b3f +-13.406,201,6.3,a3536e394b3f +-13.294,201,7.5,a3536e394b3f +-13.181,201,7.7,a3536e394b3f +-13.064,201,4.8,27de23496b0f +-12.953,201,6.0,27de23496b0f +-12.839,201,6.3,27de23496b0f +-12.731,201,4.9,27de23496b0f +-12.622,201,7.6,27de23496b0f +-12.505,201,7.3,a3536e394b3f +-12.396,201,6.3,a3536e394b3f +-12.281,201,4.1,a3536e394b3f +-12.176,201,4.4,a3536e394b3f +-12.062,201,3.4,a3536e394b3f +-11.957,201,4.3,a3536e394b3f +-11.847,201,6.4,a3536e394b3f +-11.739,201,4.8,27de23496b0f +-11.634,201,5.4,27de23496b0f +-11.52,201,5.4,27de23496b0f +-11.414,201,5.6,27de23496b0f +-11.3,201,5.8,27de23496b0f +-11.194,201,7.9,27de23496b0f +-11.084,201,6.1,27de23496b0f +-10.972,201,5.2,27de23496b0f +-10.863,201,5.6,27de23496b0f +-10.754,201,6.1,27de23496b0f +-10.644,201,5.6,27de23496b0f +-10.531,201,7.5,27de23496b0f +-10.413,201,5.3,a3536e394b3f +-10.298,201,6.4,27de23496b0f +-10.184,201,5.5,27de23496b0f +-10.073,201,7.5,27de23496b0f +-9.955,201,6.6,a3536e394b3f +-9.843,201,8.8,a3536e394b3f +-9.729,201,8.1,27de23496b0f +-9.611,201,7.5,27de23496b0f +-9.497,201,8.0,a3536e394b3f +-9.389,201,7.4,a3536e394b3f +-9.273,201,6.4,a3536e394b3f +-9.159,201,7.3,27de23496b0f +-9.048,201,7.1,27de23496b0f +-8.939,201,5.6,a3536e394b3f +-8.827,201,6.2,27de23496b0f +-8.715,201,6.7,27de23496b0f +-8.606,201,6.6,27de23496b0f +-8.49,201,5.3,27de23496b0f +-8.38,201,5.1,27de23496b0f +-8.265,201,6.2,a3536e394b3f +-8.157,201,7.5,a3536e394b3f +-8.039,201,7.5,a3536e394b3f +-7.922,201,7.0,27de23496b0f +-7.806,201,8.2,a3536e394b3f +-7.689,201,6.8,27de23496b0f +-7.581,201,8.4,a3536e394b3f +-7.471,201,6.0,a3536e394b3f +-7.359,201,7.4,a3536e394b3f +-7.242,201,6.6,27de23496b0f +-7.134,201,5.3,27de23496b0f +-7.02,201,5.8,27de23496b0f +-6.906,201,6.0,a3536e394b3f +-6.8,201,5.0,27de23496b0f +-6.688,201,6.0,a3536e394b3f +-6.572,201,6.8,a3536e394b3f +-6.464,201,6.5,a3536e394b3f +-6.357,201,9.2,a3536e394b3f +-6.242,201,7.6,a3536e394b3f +-6.132,201,9.1,a3536e394b3f +-6.017,201,7.6,27de23496b0f +-5.906,201,6.5,27de23496b0f +-5.79,201,7.5,27de23496b0f +-5.679,201,7.8,a3536e394b3f +-5.561,201,6.9,a3536e394b3f +-5.444,201,7.5,27de23496b0f +-5.327,201,7.7,a3536e394b3f +-5.218,201,9.4,a3536e394b3f +-5.099,201,8.1,a3536e394b3f +-4.989,201,7.1,a3536e394b3f +-4.875,201,6.2,27de23496b0f +-4.764,201,5.4,27de23496b0f +-4.65,201,5.8,27de23496b0f +-4.539,201,7.7,a3536e394b3f +-4.431,201,6.4,27de23496b0f +-4.319,201,8.2,a3536e394b3f +-4.203,201,7.8,a3536e394b3f +-4.091,201,6.8,a3536e394b3f +-3.983,201,6.0,a3536e394b3f +-3.874,201,6.6,a3536e394b3f +-3.761,201,6.2,a3536e394b3f +-3.652,201,5.9,a3536e394b3f +-3.544,201,6.3,a3536e394b3f +-3.43,201,6.7,a3536e394b3f +-3.323,201,5.9,a3536e394b3f +-3.207,201,7.6,27de23496b0f +-3.098,201,7.5,27de23496b0f +-2.981,201,5.6,27de23496b0f +-2.871,201,5.3,27de23496b0f +-2.756,201,5.8,a3536e394b3f +-2.642,201,5.2,a3536e394b3f +-2.531,201,7.0,a3536e394b3f +-2.422,201,6.2,a3536e394b3f +-2.314,201,5.4,a3536e394b3f +-2.206,201,6.2,a3536e394b3f +-2.089,201,6.9,a3536e394b3f +-1.98,201,3.7,a3536e394b3f +-1.875,201,5.0,a3536e394b3f +-1.768,201,5.1,a3536e394b3f +-1.652,201,6.7,a3536e394b3f +-1.537,201,5.0,a3536e394b3f +-1.424,201,5.3,27de23496b0f +-1.316,201,7.0,27de23496b0f +-1.199,201,8.6,a3536e394b3f +-1.081,201,7.3,a3536e394b3f +-0.972,201,6.3,a3536e394b3f +-0.856,201,6.2,27de23496b0f +-0.74,201,6.5,27de23496b0f +-0.632,201,4.8,27de23496b0f +-0.526,201,5.7,27de23496b0f +-0.412,201,5.6,27de23496b0f +-0.306,201,4.3,27de23496b0f +-0.197,201,5.4,27de23496b0f +-0.089,201,5.7,27de23496b0f +0.019,201,2.9,a3536e394b3f +0.125,201,4.8,a3536e394b3f +0.236,201,5.8,a3536e394b3f +0.347,201,7.3,a3536e394b3f +0.461,201,5.8,a3536e394b3f +0.577,201,6.4,27de23496b0f +0.691,201,7.2,a3536e394b3f +0.802,201,7.5,a3536e394b3f +0.916,201,9.0,a3536e394b3f +1.035,201,6.3,a3536e394b3f +1.151,201,5.5,a3536e394b3f +1.267,201,5.1,a3536e394b3f +1.373,201,5.8,a3536e394b3f +1.489,201,6.0,a3536e394b3f +1.598,201,6.7,a3536e394b3f +1.707,201,6.4,a3536e394b3f +1.822,201,7.8,a3536e394b3f +1.935,201,8.4,a3536e394b3f +2.053,201,6.8,a3536e394b3f +2.16,201,7.5,a3536e394b3f +2.269,201,6.2,a3536e394b3f +2.385,201,5.5,a3536e394b3f +2.5,201,5.0,a3536e394b3f +2.611,201,5.2,a3536e394b3f +2.722,201,5.7,a3536e394b3f +2.832,201,6.5,a3536e394b3f +2.948,201,5.9,a3536e394b3f +3.059,201,6.7,a3536e394b3f +3.168,201,5.4,a3536e394b3f +3.282,201,5.8,a3536e394b3f +3.389,201,5.3,a3536e394b3f +3.501,201,5.7,a3536e394b3f +3.616,201,7.3,a3536e394b3f +3.733,201,7.2,a3536e394b3f +3.844,201,8.4,a3536e394b3f +3.961,201,8.1,a3536e394b3f +4.078,201,7.5,a3536e394b3f +4.186,201,7.9,a3536e394b3f +4.298,201,6.8,a3536e394b3f +4.405,201,6.7,a3536e394b3f +4.522,201,7.4,a3536e394b3f +4.64,201,8.1,a3536e394b3f +4.751,201,7.9,a3536e394b3f +4.861,201,7.7,a3536e394b3f +4.969,201,7.4,a3536e394b3f +5.081,201,7.7,a3536e394b3f +5.189,201,9.2,a3536e394b3f +5.306,201,5.5,a3536e394b3f +5.42,201,6.5,a3536e394b3f +5.536,201,4.3,a3536e394b3f +5.647,201,5.2,a3536e394b3f +5.757,201,7.0,a3536e394b3f +5.869,201,5.9,a3536e394b3f +5.985,201,7.8,a3536e394b3f +6.097,201,7.6,a3536e394b3f +6.211,201,6.8,a3536e394b3f +6.326,201,5.4,a3536e394b3f +6.441,201,5.9,a3536e394b3f +6.557,201,7.4,a3536e394b3f +6.672,201,7.6,a3536e394b3f +6.786,201,7.0,a3536e394b3f +6.904,201,6.4,a3536e394b3f +7.017,201,7.8,a3536e394b3f +7.128,201,7.4,a3536e394b3f +7.242,201,8.9,a3536e394b3f +7.353,201,5.7,a3536e394b3f +7.459,201,5.2,a3536e394b3f +7.572,201,5.5,a3536e394b3f +7.678,201,6.5,a3536e394b3f +7.793,201,7.0,a3536e394b3f +7.907,201,7.3,a3536e394b3f +8.016,201,6.8,a3536e394b3f +8.128,201,6.9,a3536e394b3f +8.244,201,7.0,a3536e394b3f +8.353,201,4.6,a3536e394b3f +8.464,201,4.7,a3536e394b3f +8.57,201,4.3,a3536e394b3f +8.684,201,6.1,a3536e394b3f +8.794,201,6.3,a3536e394b3f +8.901,201,7.9,a3536e394b3f +9.015,201,8.6,a3536e394b3f +9.134,201,7.7,a3536e394b3f +9.244,201,6.8,a3536e394b3f +9.361,201,5.9,a3536e394b3f +9.474,201,6.7,a3536e394b3f +9.591,201,6.6,a3536e394b3f +9.699,201,6.2,a3536e394b3f +9.808,201,6.4,a3536e394b3f +9.921,201,8.1,a3536e394b3f +10.037,201,7.7,a3536e394b3f +10.148,201,5.9,a3536e394b3f +10.255,201,7.7,a3536e394b3f +10.373,201,8.4,a3536e394b3f +10.491,201,9.0,a3536e394b3f +10.604,201,7.0,a3536e394b3f +10.719,201,11.9,a3536e394b3f +10.836,201,6.8,a3536e394b3f +10.944,201,8.6,a3536e394b3f +11.061,201,8.4,a3536e394b3f +11.178,201,6.9,a3536e394b3f +11.287,201,6.1,a3536e394b3f +11.403,201,5.7,a3536e394b3f +11.513,201,6.4,a3536e394b3f +11.628,201,5.1,a3536e394b3f +11.736,201,7.0,a3536e394b3f +11.844,201,6.6,a3536e394b3f +11.955,201,5.5,a3536e394b3f +12.061,201,10.1,a3536e394b3f +12.178,201,8.6,a3536e394b3f +12.29,201,8.2,a3536e394b3f +12.403,201,7.3,a3536e394b3f +12.518,201,8.4,a3536e394b3f +12.632,201,6.6,a3536e394b3f +12.741,201,3.8,a3536e394b3f +12.851,201,3.3,a3536e394b3f +12.956,201,6.6,a3536e394b3f +13.07,201,4.5,a3536e394b3f +13.178,201,3.8,a3536e394b3f +13.286,201,4.4,a3536e394b3f +13.395,201,3.9,a3536e394b3f +13.509,201,4.1,a3536e394b3f +13.616,201,7.0,a3536e394b3f +13.731,201,4.1,a3536e394b3f +13.836,201,6.0,a3536e394b3f +13.952,201,3.1,a3536e394b3f +14.056,201,6.6,a3536e394b3f +14.17,201,8.8,a3536e394b3f +14.284,201,6.1,a3536e394b3f +14.39,201,6.3,a3536e394b3f +14.501,201,4.1,a3536e394b3f +14.606,201,6.1,a3536e394b3f +14.718,201,7.9,a3536e394b3f +14.831,201,7.7,a3536e394b3f +14.948,201,5.8,27de23496b0f +15.062,201,6.6,a3536e394b3f +15.18,201,5.4,a3536e394b3f +15.295,201,6.2,27de23496b0f +15.411,201,5.5,27de23496b0f +15.519,201,5.2,27de23496b0f +15.63,201,5.0,27de23496b0f +15.737,201,5.4,27de23496b0f +15.849,201,7.0,27de23496b0f +15.966,201,6.5,a3536e394b3f +16.072,201,6.3,a3536e394b3f +16.181,201,7.6,27de23496b0f +16.29,201,8.2,27de23496b0f +16.407,201,6.9,a3536e394b3f +16.515,201,5.8,a3536e394b3f +16.623,201,7.0,a3536e394b3f +16.733,201,6.0,a3536e394b3f +16.84,201,7.6,27de23496b0f +16.955,201,5.9,a3536e394b3f +17.061,201,8.0,a3536e394b3f +17.178,201,8.3,a3536e394b3f +17.288,201,7.9,a3536e394b3f +17.404,201,8.5,27de23496b0f +17.514,201,5.8,27de23496b0f +17.623,201,6.5,27de23496b0f +17.736,201,5.2,27de23496b0f +17.842,201,4.3,a3536e394b3f +17.95,201,11.5,27de23496b0f +18.069,201,5.6,27de23496b0f +18.185,201,6.9,a3536e394b3f +18.293,201,8.1,a3536e394b3f +18.411,201,6.2,a3536e394b3f +18.519,201,7.6,a3536e394b3f +18.628,201,5.8,a3536e394b3f +18.736,201,4.3,a3536e394b3f +18.841,201,6.6,a3536e394b3f +18.95,201,6.2,27de23496b0f +19.057,201,5.7,27de23496b0f +19.166,201,4.2,27de23496b0f +19.278,201,7.4,27de23496b0f +19.393,201,5.7,a3536e394b3f +19.507,201,5.7,a3536e394b3f +19.619,201,6.0,a3536e394b3f +19.728,201,4.8,a3536e394b3f +19.841,201,6.1,a3536e394b3f +19.957,201,7.1,27de23496b0f +20.066,201,6.4,27de23496b0f +20.182,201,7.4,a3536e394b3f +20.298,201,7.7,a3536e394b3f +20.415,201,6.8,27de23496b0f +20.532,201,7.6,a3536e394b3f +20.644,201,8.7,a3536e394b3f +20.755,201,6.0,a3536e394b3f +20.868,201,7.0,a3536e394b3f +20.984,201,6.8,27de23496b0f +21.094,201,7.3,27de23496b0f +21.212,201,7.3,a3536e394b3f +21.322,201,6.4,a3536e394b3f +21.435,201,6.0,a3536e394b3f +21.546,201,7.3,a3536e394b3f +21.663,201,7.4,27de23496b0f +21.778,201,5.0,27de23496b0f +21.886,201,6.8,27de23496b0f +22.001,201,7.7,27de23496b0f +22.109,201,6.8,27de23496b0f +22.216,201,7.7,27de23496b0f +22.332,201,9.8,27de23496b0f +22.448,201,4.8,a3536e394b3f +22.562,201,6.2,a3536e394b3f +22.672,201,6.9,27de23496b0f +22.781,201,10.3,27de23496b0f +22.894,201,8.0,27de23496b0f +23.011,201,8.4,a3536e394b3f +23.13,201,9.3,27de23496b0f +23.241,201,8.9,27de23496b0f +23.352,201,7.8,27de23496b0f +23.47,201,8.3,27de23496b0f +23.586,201,5.6,a3536e394b3f +23.702,201,6.2,27de23496b0f +23.814,201,5.6,27de23496b0f +23.919,201,5.1,a3536e394b3f +24.031,201,5.7,27de23496b0f +24.14,201,5.2,a3536e394b3f +24.253,201,4.2,27de23496b0f +24.363,201,6.2,27de23496b0f +24.478,201,6.4,27de23496b0f +24.593,201,6.9,a3536e394b3f +24.702,201,6.2,27de23496b0f +24.814,201,8.7,a3536e394b3f +24.928,201,7.3,a3536e394b3f +25.045,201,7.2,27de23496b0f +25.157,201,6.3,27de23496b0f +25.264,201,7.6,27de23496b0f +25.379,201,6.7,27de23496b0f +25.494,201,7.1,a3536e394b3f +25.602,201,8.1,27de23496b0f +25.711,201,6.3,a3536e394b3f +25.827,201,3.9,a3536e394b3f +25.936,201,5.4,a3536e394b3f +26.049,201,7.5,a3536e394b3f +26.165,201,7.5,27de23496b0f +26.282,201,7.1,a3536e394b3f +26.395,201,5.7,a3536e394b3f +26.503,201,6.7,a3536e394b3f +26.615,201,6.9,a3536e394b3f +26.726,201,6.8,a3536e394b3f +26.84,201,7.0,a3536e394b3f +26.955,201,7.5,27de23496b0f +27.069,201,6.3,27de23496b0f +27.185,201,6.3,a3536e394b3f +27.294,201,7.7,a3536e394b3f +27.41,201,6.2,a3536e394b3f +27.526,201,7.2,27de23496b0f +27.643,201,8.0,a3536e394b3f +27.761,201,8.1,27de23496b0f +27.879,201,5.8,27de23496b0f +27.986,201,5.0,27de23496b0f +28.096,201,6.4,27de23496b0f +28.212,201,9.2,a3536e394b3f +28.328,201,6.8,a3536e394b3f +28.445,201,5.5,27de23496b0f +28.559,201,5.7,27de23496b0f +28.672,201,7.3,a3536e394b3f +28.781,201,5.6,27de23496b0f +28.896,201,7.6,a3536e394b3f +29.008,201,5.5,a3536e394b3f +29.114,201,5.8,a3536e394b3f +29.228,201,7.2,a3536e394b3f +29.344,201,5.2,27de23496b0f +29.453,201,5.7,27de23496b0f +29.561,201,5.7,27de23496b0f +29.669,201,6.7,a3536e394b3f +29.783,201,6.1,27de23496b0f +29.894,201,6.4,27de23496b0f +30.01,201,6.0,27de23496b0f +30.122,201,7.0,a3536e394b3f +30.231,201,6.4,27de23496b0f +30.34,201,8.7,27de23496b0f +30.455,201,5.7,a3536e394b3f +30.57,201,7.1,a3536e394b3f +30.685,201,6.9,27de23496b0f +30.795,201,6.0,27de23496b0f +30.911,201,7.8,a3536e394b3f +31.029,201,7.7,a3536e394b3f +31.145,201,6.3,27de23496b0f +31.261,201,7.7,a3536e394b3f +31.369,201,6.2,27de23496b0f +31.485,201,6.5,a3536e394b3f +31.594,201,7.5,a3536e394b3f +31.703,201,5.6,a3536e394b3f +31.812,201,7.0,a3536e394b3f +31.928,201,9.1,a3536e394b3f +32.044,201,6.3,27de23496b0f +32.152,201,6.3,27de23496b0f +32.262,201,6.8,a3536e394b3f +32.369,201,5.6,a3536e394b3f +32.476,201,5.4,a3536e394b3f +32.582,201,5.3,a3536e394b3f +32.696,201,8.3,a3536e394b3f +32.805,201,7.1,a3536e394b3f +32.923,201,7.3,27de23496b0f +33.04,201,6.1,27de23496b0f +33.147,201,6.4,27de23496b0f +33.261,201,6.7,27de23496b0f +33.376,201,6.3,a3536e394b3f +33.483,201,7.1,a3536e394b3f +33.597,201,6.5,a3536e394b3f +33.706,201,7.1,a3536e394b3f +33.814,201,5.3,a3536e394b3f +33.93,201,6.7,a3536e394b3f +34.038,201,4.9,a3536e394b3f +34.143,201,4.1,a3536e394b3f +34.256,201,4.3,a3536e394b3f +34.369,201,5.1,a3536e394b3f +34.475,201,5.5,a3536e394b3f +34.589,201,4.6,a3536e394b3f +34.694,201,3.8,a3536e394b3f +34.801,201,6.7,a3536e394b3f +34.909,201,7.1,27de23496b0f +35.021,201,8.7,a3536e394b3f +35.14,201,7.2,a3536e394b3f +35.252,201,7.2,27de23496b0f +35.361,201,7.7,27de23496b0f +35.478,201,6.5,27de23496b0f +35.586,201,7.1,27de23496b0f +35.703,201,6.9,a3536e394b3f +35.817,201,6.1,a3536e394b3f +35.93,201,5.5,a3536e394b3f +36.046,201,4.9,27de23496b0f +36.161,201,6.9,a3536e394b3f +36.278,201,7.2,a3536e394b3f +36.394,201,6.6,27de23496b0f +36.509,201,7.1,27de23496b0f +36.627,201,6.0,a3536e394b3f +36.743,201,7.6,27de23496b0f +36.856,201,8.4,27de23496b0f +36.973,201,7.9,a3536e394b3f +37.085,201,7.5,a3536e394b3f +37.193,201,8.0,a3536e394b3f +37.309,201,7.2,a3536e394b3f +37.422,201,7.3,27de23496b0f +37.537,201,8.2,27de23496b0f +37.653,201,7.2,a3536e394b3f +37.761,201,8.5,a3536e394b3f +37.88,201,5.7,a3536e394b3f +37.995,201,5.5,27de23496b0f +38.103,201,5.9,27de23496b0f +38.212,201,5.9,a3536e394b3f +38.318,201,8.7,a3536e394b3f +38.435,201,8.2,27de23496b0f +38.551,201,6.4,27de23496b0f +38.657,201,7.1,27de23496b0f +38.765,201,6.5,27de23496b0f +38.878,201,9.3,27de23496b0f +38.996,201,6.6,a3536e394b3f +39.113,201,7.1,27de23496b0f +39.226,201,6.0,27de23496b0f +39.341,201,3.4,27de23496b0f +39.451,201,3.9,27de23496b0f +39.558,201,4.2,27de23496b0f +39.668,201,4.1,27de23496b0f +39.78,201,4.6,27de23496b0f +39.894,201,5.0,a3536e394b3f +40.008,201,5.4,a3536e394b3f +40.119,201,5.6,a3536e394b3f +40.227,201,5.1,a3536e394b3f +40.338,201,6.9,a3536e394b3f +40.452,201,6.1,a3536e394b3f +40.567,201,5.8,27de23496b0f +40.683,201,7.4,a3536e394b3f +40.793,201,7.1,a3536e394b3f +40.909,201,7.9,a3536e394b3f +41.019,201,6.7,a3536e394b3f +41.128,201,7.0,a3536e394b3f +41.236,201,6.9,a3536e394b3f +41.352,201,6.5,27de23496b0f +41.466,201,6.2,27de23496b0f +41.578,201,6.1,27de23496b0f +41.692,201,7.5,a3536e394b3f +41.803,201,7.4,a3536e394b3f +41.916,201,9.2,a3536e394b3f +42.031,201,5.6,27de23496b0f +42.146,201,5.0,27de23496b0f +42.261,201,6.0,27de23496b0f +42.369,201,5.9,27de23496b0f +42.482,201,7.0,a3536e394b3f +42.594,201,5.9,a3536e394b3f +42.703,201,7.0,a3536e394b3f +42.815,201,6.8,a3536e394b3f +42.928,201,6.9,a3536e394b3f +43.042,201,6.5,27de23496b0f +43.149,201,13.4,a3536e394b3f +43.271,201,8.6,a3536e394b3f +43.39,201,6.3,a3536e394b3f +43.504,201,7.1,27de23496b0f +43.62,201,6.8,27de23496b0f +43.734,201,6.4,a3536e394b3f +43.848,201,6.2,a3536e394b3f +43.964,201,8.1,27de23496b0f +44.078,201,4.9,27de23496b0f +44.19,201,6.1,27de23496b0f +44.297,201,5.9,27de23496b0f +44.41,201,4.4,27de23496b0f +44.524,201,4.2,a3536e394b3f +44.632,201,4.8,a3536e394b3f +44.743,201,7.1,a3536e394b3f +44.853,201,5.1,a3536e394b3f +44.965,201,4.3,a3536e394b3f +45.079,201,6.3,a3536e394b3f +45.19,201,10.8,27de23496b0f +45.301,201,12.5,a3536e394b3f +45.414,201,13.5,27de23496b0f +45.529,201,5.7,a3536e394b3f diff --git a/evidence/failover-each/27de23496b0f/result.json b/evidence/failover-each/27de23496b0f/result.json new file mode 100644 index 00000000..68ba9626 --- /dev/null +++ b/evidence/failover-each/27de23496b0f/result.json @@ -0,0 +1,60 @@ +{ + "label": "27de23496b0f", + "action": [ + "sh", + "-c", + "scripts/stop-one.sh && sleep 10 && scripts/restore.sh" + ], + "action_exit": 0, + "action_seconds": 14.508, + "action_finished_before_load_end": true, + "lb_up_before": 2, + "lb_up_min": 1, + "detection_seconds": 0.894, + "rejoin_seconds": 14.392, + "probe": { + "attempts": 635, + "failed_attempts": 0, + "first_failure_s": null, + "recovery_s": 0.0, + "instances_before": { + "a3536e394b3f": 118, + "27de23496b0f": 111 + }, + "instances_during_outage": { + "a3536e394b3f": 119 + }, + "instances_after": { + "a3536e394b3f": 152, + "27de23496b0f": 124 + } + }, + "lb_counters": { + "wretr": 0, + "wredis": 0, + "econ": 0, + "eresp": 0, + "hrsp_5xx": 0 + }, + "k6": { + "requests": 6001, + "rps": 100.02, + "ok_rps": 100.02, + "failed_ratio": 0, + "dropped_iterations": 0, + "p50_ms": 2.44, + "p95_ms": 3.42, + "p99_ms": 5.08, + "exit": 0, + "rate": 100 + }, + "checks": { + "action_ok": true, + "load_errors_ok": true, + "write_recovery_ok": true, + "detected": true, + "target_excluded": true, + "rejoined": true + }, + "pass": true +} diff --git a/evidence/failover-each/a3536e394b3f/action.txt b/evidence/failover-each/a3536e394b3f/action.txt new file mode 100644 index 00000000..78cf725a --- /dev/null +++ b/evidence/failover-each/a3536e394b3f/action.txt @@ -0,0 +1,7 @@ + Container mtusi2026lab3-api-2 Running + Container mtusi2026lab3-api-1 Starting + Container mtusi2026lab3-api-1 Started + Container mtusi2026lab3-api-2 Waiting + Container mtusi2026lab3-api-1 Waiting + Container mtusi2026lab3-api-2 Healthy + Container mtusi2026lab3-api-1 Healthy diff --git a/evidence/failover-each/a3536e394b3f/k6/k6.txt b/evidence/failover-each/a3536e394b3f/k6/k6.txt new file mode 100644 index 00000000..631b27d1 --- /dev/null +++ b/evidence/failover-each/a3536e394b3f/k6/k6.txt @@ -0,0 +1,2 @@ +time="2026-10-28T16:58:20Z" level=info msg=LOAD-START source=console +measure: 6001 requests, failed ratio 0 diff --git a/evidence/failover-each/a3536e394b3f/k6/summary.json b/evidence/failover-each/a3536e394b3f/k6/summary.json new file mode 100644 index 00000000..5c6081b9 --- /dev/null +++ b/evidence/failover-each/a3536e394b3f/k6/summary.json @@ -0,0 +1,309 @@ +{ + "root_group": { + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "fails": 0, + "name": "message 201", + "path": "::message 201", + "id": "6445aff337e1ade0bc10402fbac79882", + "passes": 5609 + }, + { + "name": "history 200", + "path": "::history 200", + "id": "fe27b3b8edd292c7f2170a6357b087d0", + "passes": 1393, + "fails": 0 + } + ], + "name": "" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "p(90)", + "p(95)", + "p(99)", + "max" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "testRunDurationMs": 70492.415338, + "isStdOutTTY": false, + "isStdErrTTY": false + }, + "metrics": { + "http_req_failed{scenario:measure}": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 6001 + } + }, + "http_req_duration{expected_response:true}": { + "values": { + "p(99)": 5.622045839999999, + "max": 33.685, + "avg": 2.477255534032903, + "min": 0.884, + "med": 2.395354, + "p(90)": 3.1186628, + "p(95)": 3.4812878 + }, + "type": "trend", + "contains": "time" + }, + "http_req_receiving": { + "contains": "time", + "values": { + "max": 1.061792, + "avg": 0.029622949659671046, + "min": 0.008, + "med": 0.026542, + "p(90)": 0.04141190000000001, + "p(95)": 0.05360189999999999, + "p(99)": 0.09724916 + }, + "type": "trend" + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 0.947709, + "med": 2.4944585, + "p(90)": 3.2278125, + "p(95)": 3.546753499999998, + "p(99)": 5.4046430899999995, + "max": 33.766792, + "avg": 2.5308004734361425 + } + }, + "http_req_duration{scenario:measure,name:get_history}": { + "values": { + "p(95)": 3.2932419, + "p(99)": 4.692470499999999, + "max": 33.685, + "avg": 2.351975028451884, + "min": 1.041667, + "med": 2.326709, + "p(90)": 2.9685998000000002 + }, + "type": "trend", + "contains": "time" + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 7052 + } + }, + "http_reqs{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "rate": 85.12972596024909, + "count": 6001 + } + }, + "dropped_iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 0, + "rate": 0 + } + }, + "http_req_blocked": { + "contains": "time", + "values": { + "avg": 0.003003323596142927, + "min": 0.000625, + "med": 0.001917, + "p(90)": 0.002875, + "p(95)": 0.003708, + "p(99)": 0.01333349, + "max": 0.522375 + }, + "type": "trend" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.01813603062960856, + "min": 0.00225, + "med": 0.015875, + "p(90)": 0.021333, + "p(95)": 0.03156044999999999, + "p(99)": 0.07835074999999998, + "max": 0.605584 + } + }, + "data_received": { + "contains": "data", + "values": { + "count": 17128543, + "rate": 242984.1979150713 + }, + "type": "counter" + }, + "http_req_duration": { + "contains": "time", + "values": { + "avg": 2.477255534032903, + "min": 0.884, + "med": 2.395354, + "p(90)": 3.1186628, + "p(95)": 3.4812878, + "p(99)": 5.622045839999999, + "max": 33.685 + }, + "type": "trend" + }, + "iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 6001, + "rate": 85.12972596024909 + } + }, + "http_req_duration{scenario:measure,name:post_message}": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 3.115729, + "p(95)": 3.41746875, + "p(99)": 5.243429499999999, + "max": 18.370209, + "avg": 2.4588117602996236, + "min": 0.964083, + "med": 2.4317915 + } + }, + "checks": { + "values": { + "rate": 1, + "passes": 7002, + "fails": 0 + }, + "type": "rate", + "contains": "default" + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "max": 0, + "value": 0, + "min": 0 + } + }, + "http_req_tls_handshaking": { + "contains": "time", + "values": { + "p(99)": 0, + "max": 0, + "avg": 0, + "min": 0, + "med": 0, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 7052, + "rate": 100.03913139004776 + } + }, + "http_req_duration{scenario:measure}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 2.4375369903349426, + "min": 0.964083, + "med": 2.402375, + "p(90)": 3.098583, + "p(95)": 3.407, + "p(99)": 5.098333, + "max": 33.685 + } + }, + "dropped_iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 0, + "rate": 0 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "p(90)": 0, + "p(95)": 0, + "p(99)": 0, + "max": 0.380625, + "avg": 0.0005381395348837208, + "min": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 2052688, + "rate": 29119.274607880652 + } + }, + "vus_max": { + "contains": "default", + "values": { + "min": 20, + "max": 20, + "value": 20 + }, + "type": "gauge" + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "rate": 99.32983522307352, + "count": 7002 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "max": 33.630417, + "avg": 2.429496553743612, + "min": 0.848083, + "med": 2.3483125, + "p(90)": 3.0558042, + "p(95)": 3.4048060999999996, + "p(99)": 5.55202933 + } + } + } +} \ No newline at end of file diff --git a/evidence/failover-each/a3536e394b3f/lb_up.csv b/evidence/failover-each/a3536e394b3f/lb_up.csv new file mode 100644 index 00000000..a6422db4 --- /dev/null +++ b/evidence/failover-each/a3536e394b3f/lb_up.csv @@ -0,0 +1,644 @@ +t_rel_s,up +-25.738,2 +-25.631,2 +-25.526,2 +-25.42,2 +-25.308,2 +-25.197,2 +-25.086,2 +-24.971,2 +-24.857,2 +-24.745,2 +-24.638,2 +-24.527,2 +-24.424,2 +-24.316,2 +-24.211,2 +-24.097,2 +-23.984,2 +-23.87,2 +-23.757,2 +-23.645,2 +-23.541,2 +-23.426,2 +-23.313,2 +-23.197,2 +-23.083,2 +-22.977,2 +-22.868,2 +-22.76,2 +-22.646,2 +-22.54,2 +-22.428,2 +-22.318,2 +-22.203,2 +-22.087,2 +-21.972,2 +-21.859,2 +-21.745,2 +-21.636,2 +-21.525,2 +-21.41,2 +-21.302,2 +-21.187,2 +-21.072,2 +-20.956,2 +-20.841,2 +-20.728,2 +-20.62,2 +-20.514,2 +-20.409,2 +-20.302,2 +-20.195,2 +-20.08,2 +-19.975,2 +-19.861,2 +-19.746,2 +-19.64,2 +-19.527,2 +-19.414,2 +-19.309,2 +-19.193,2 +-19.087,2 +-18.98,2 +-18.868,2 +-18.762,2 +-18.656,2 +-18.55,2 +-18.442,2 +-18.326,2 +-18.211,2 +-18.098,2 +-17.987,2 +-17.877,2 +-17.771,2 +-17.665,2 +-17.56,2 +-17.452,2 +-17.346,2 +-17.232,2 +-17.125,2 +-17.019,2 +-16.911,2 +-16.802,2 +-16.698,2 +-16.594,2 +-16.479,2 +-16.371,2 +-16.26,2 +-16.145,2 +-16.028,2 +-15.916,2 +-15.804,2 +-15.697,2 +-15.592,2 +-15.487,2 +-15.381,2 +-15.269,2 +-15.161,2 +-15.046,2 +-14.929,2 +-14.815,2 +-14.71,2 +-14.604,2 +-14.49,2 +-14.377,2 +-14.263,2 +-14.147,2 +-14.039,2 +-13.924,2 +-13.811,2 +-13.7,2 +-13.584,2 +-13.476,2 +-13.36,2 +-13.25,2 +-13.136,2 +-13.027,2 +-12.914,2 +-12.799,2 +-12.683,2 +-12.577,2 +-12.469,2 +-12.356,2 +-12.252,2 +-12.139,2 +-12.028,2 +-11.922,2 +-11.818,2 +-11.704,2 +-11.594,2 +-11.49,2 +-11.38,2 +-11.272,2 +-11.161,2 +-11.046,2 +-10.932,2 +-10.824,2 +-10.714,2 +-10.606,2 +-10.492,2 +-10.387,2 +-10.277,2 +-10.171,2 +-10.056,2 +-9.947,2 +-9.835,2 +-9.722,2 +-9.606,2 +-9.491,2 +-9.377,2 +-9.261,2 +-9.147,2 +-9.041,2 +-8.936,2 +-8.827,2 +-8.72,2 +-8.61,2 +-8.501,2 +-8.394,2 +-8.279,2 +-8.162,2 +-8.051,2 +-7.939,2 +-7.827,2 +-7.719,2 +-7.605,2 +-7.494,2 +-7.381,2 +-7.271,2 +-7.159,2 +-7.049,2 +-6.937,2 +-6.822,2 +-6.711,2 +-6.596,2 +-6.48,2 +-6.369,2 +-6.255,2 +-6.144,2 +-6.036,2 +-5.921,2 +-5.805,2 +-5.692,2 +-5.587,2 +-5.472,2 +-5.358,2 +-5.252,2 +-5.144,2 +-5.038,2 +-4.923,2 +-4.815,2 +-4.708,2 +-4.602,2 +-4.487,2 +-4.38,2 +-4.269,2 +-4.156,2 +-4.046,2 +-3.931,2 +-3.82,2 +-3.702,2 +-3.593,2 +-3.478,2 +-3.37,2 +-3.261,2 +-3.152,2 +-3.036,2 +-2.928,2 +-2.813,2 +-2.702,2 +-2.589,2 +-2.482,2 +-2.366,2 +-2.248,2 +-2.138,2 +-2.027,2 +-1.913,2 +-1.802,2 +-1.694,2 +-1.579,2 +-1.469,2 +-1.358,2 +-1.252,2 +-1.139,2 +-1.03,2 +-0.92,2 +-0.811,2 +-0.695,2 +-0.581,2 +-0.474,2 +-0.36,2 +-0.248,2 +-0.138,2 +-0.022,2 +0.09,2 +0.194,2 +0.3,2 +0.404,2 +0.514,2 +0.626,2 +0.731,2 +0.839,2 +0.946,2 +1.063,2 +1.173,2 +1.284,2 +1.399,2 +1.515,1 +1.628,1 +1.736,1 +1.852,1 +1.967,1 +2.072,1 +2.186,1 +2.297,1 +2.412,1 +2.527,1 +2.641,1 +2.747,1 +2.861,1 +2.973,1 +3.086,1 +3.2,1 +3.317,1 +3.421,1 +3.535,1 +3.646,1 +3.755,1 +3.871,1 +3.983,1 +4.089,1 +4.204,1 +4.314,1 +4.419,1 +4.525,1 +4.64,1 +4.755,1 +4.862,1 +4.978,1 +5.089,1 +5.206,1 +5.316,1 +5.432,1 +5.54,1 +5.648,1 +5.761,1 +5.868,1 +5.981,1 +6.095,1 +6.207,1 +6.323,1 +6.43,1 +6.537,1 +6.651,1 +6.765,1 +6.872,1 +6.978,1 +7.093,1 +7.205,1 +7.313,1 +7.422,1 +7.531,1 +7.642,1 +7.752,1 +7.864,1 +7.98,1 +8.098,1 +8.214,1 +8.323,1 +8.438,1 +8.553,1 +8.664,1 +8.773,1 +8.887,1 +8.992,1 +9.107,1 +9.214,1 +9.319,1 +9.429,1 +9.536,1 +9.642,1 +9.756,1 +9.871,1 +9.987,1 +10.093,1 +10.206,1 +10.315,1 +10.426,1 +10.532,1 +10.644,1 +10.754,1 +10.869,1 +10.975,1 +11.089,1 +11.203,1 +11.314,1 +11.422,1 +11.533,1 +11.649,1 +11.764,1 +11.88,1 +11.994,1 +12.109,1 +12.218,1 +12.33,1 +12.441,1 +12.548,1 +12.661,1 +12.77,1 +12.874,1 +12.987,1 +13.098,1 +13.21,1 +13.323,1 +13.438,1 +13.553,1 +13.664,1 +13.77,1 +13.884,1 +13.999,1 +14.111,2 +14.222,2 +14.326,2 +14.435,2 +14.546,2 +14.66,2 +14.766,2 +14.881,2 +14.992,2 +15.103,2 +15.218,2 +15.326,2 +15.439,2 +15.548,2 +15.659,2 +15.769,2 +15.879,2 +15.987,2 +16.105,2 +16.214,2 +16.331,2 +16.442,2 +16.554,2 +16.661,2 +16.767,2 +16.876,2 +16.986,2 +17.097,2 +17.203,2 +17.314,2 +17.42,2 +17.536,2 +17.647,2 +17.756,2 +17.863,2 +17.968,2 +18.085,2 +18.201,2 +18.316,2 +18.433,2 +18.549,2 +18.664,2 +18.776,2 +18.885,2 +18.992,2 +19.106,2 +19.219,2 +19.331,2 +19.446,2 +19.559,2 +19.665,2 +19.78,2 +19.894,2 +20.0,2 +20.117,2 +20.225,2 +20.331,2 +20.441,2 +20.557,2 +20.673,2 +20.79,2 +20.897,2 +21.009,2 +21.122,2 +21.237,2 +21.35,2 +21.466,2 +21.581,2 +21.696,2 +21.801,2 +21.915,2 +22.025,2 +22.131,2 +22.246,2 +22.36,2 +22.475,2 +22.582,2 +22.698,2 +22.805,2 +22.914,2 +23.029,2 +23.142,2 +23.248,2 +23.365,2 +23.481,2 +23.591,2 +23.699,2 +23.815,2 +23.931,2 +24.047,2 +24.161,2 +24.274,2 +24.38,2 +24.485,2 +24.598,2 +24.706,2 +24.813,2 +24.927,2 +25.037,2 +25.15,2 +25.256,2 +25.363,2 +25.475,2 +25.594,2 +25.705,2 +25.82,2 +25.934,2 +26.041,2 +26.148,2 +26.256,2 +26.364,2 +26.479,2 +26.587,2 +26.703,2 +26.809,2 +26.916,2 +27.023,2 +27.138,2 +27.248,2 +27.363,2 +27.479,2 +27.595,2 +27.706,2 +27.814,2 +27.923,2 +28.038,2 +28.15,2 +28.266,2 +28.373,2 +28.488,2 +28.598,2 +28.713,2 +28.823,2 +28.932,2 +29.039,2 +29.143,2 +29.256,2 +29.367,2 +29.472,2 +29.582,2 +29.693,2 +29.798,2 +29.912,2 +30.02,2 +30.133,2 +30.238,2 +30.353,2 +30.458,2 +30.57,2 +30.686,2 +30.799,2 +30.914,2 +31.023,2 +31.131,2 +31.247,2 +31.362,2 +31.47,2 +31.579,2 +31.69,2 +31.806,2 +31.915,2 +32.027,2 +32.143,2 +32.26,2 +32.374,2 +32.49,2 +32.604,2 +32.719,2 +32.825,2 +32.933,2 +33.047,2 +33.158,2 +33.264,2 +33.373,2 +33.488,2 +33.603,2 +33.713,2 +33.826,2 +33.936,2 +34.047,2 +34.153,2 +34.261,2 +34.372,2 +34.481,2 +34.587,2 +34.701,2 +34.815,2 +34.929,2 +35.037,2 +35.151,2 +35.257,2 +35.368,2 +35.478,2 +35.581,2 +35.69,2 +35.807,2 +35.924,2 +36.034,2 +36.141,2 +36.255,2 +36.365,2 +36.471,2 +36.576,2 +36.687,2 +36.8,2 +36.908,2 +37.019,2 +37.132,2 +37.24,2 +37.354,2 +37.469,2 +37.582,2 +37.691,2 +37.803,2 +37.915,2 +38.023,2 +38.129,2 +38.245,2 +38.354,2 +38.47,2 +38.586,2 +38.699,2 +38.806,2 +38.915,2 +39.021,2 +39.129,2 +39.243,2 +39.357,2 +39.46,2 +39.575,2 +39.69,2 +39.799,2 +39.906,2 +40.012,2 +40.118,2 +40.232,2 +40.346,2 +40.461,2 +40.576,2 +40.686,2 +40.798,2 +40.907,2 +41.018,2 +41.131,2 +41.239,2 +41.353,2 +41.461,2 +41.568,2 +41.677,2 +41.791,2 +41.898,2 +42.007,2 +42.119,2 +42.227,2 +42.34,2 +42.448,2 +42.561,2 +42.671,2 +42.781,2 +42.897,2 +43.009,2 +43.123,2 +43.23,2 +43.344,2 +43.456,2 +43.563,2 +43.67,2 +43.781,2 +43.89,2 +44.005,2 +44.119,2 +44.233,2 +44.34,2 +44.449,2 +44.556,2 +44.665,2 +44.778,2 +44.89,2 +45.002,2 +45.117,2 +45.221,2 +45.335,2 +45.439,2 +45.544,2 diff --git a/evidence/failover-each/a3536e394b3f/probe.csv b/evidence/failover-each/a3536e394b3f/probe.csv new file mode 100644 index 00000000..994731f2 --- /dev/null +++ b/evidence/failover-each/a3536e394b3f/probe.csv @@ -0,0 +1,636 @@ +t_rel_s,status,latency_ms,instance +-25.738,201,16.8,a3536e394b3f +-25.621,201,8.4,27de23496b0f +-25.512,201,17.6,a3536e394b3f +-25.384,201,4.4,a3536e394b3f +-25.275,201,4.9,27de23496b0f +-25.169,201,5.0,a3536e394b3f +-25.054,201,5.7,27de23496b0f +-24.938,201,3.9,27de23496b0f +-24.824,201,4.5,27de23496b0f +-24.72,201,4.9,a3536e394b3f +-24.61,201,5.3,a3536e394b3f +-24.504,201,7.3,a3536e394b3f +-24.396,201,4.7,a3536e394b3f +-24.291,201,5.3,27de23496b0f +-24.175,201,3.8,a3536e394b3f +-24.064,201,5.0,a3536e394b3f +-23.958,201,4.5,a3536e394b3f +-23.844,201,4.4,a3536e394b3f +-23.737,201,4.7,a3536e394b3f +-23.626,201,5.0,a3536e394b3f +-23.518,201,5.7,a3536e394b3f +-23.412,201,5.9,27de23496b0f +-23.296,201,6.7,a3536e394b3f +-23.188,201,6.4,a3536e394b3f +-23.081,201,5.7,27de23496b0f +-22.969,201,5.2,27de23496b0f +-22.858,201,4.2,a3536e394b3f +-22.744,201,6.2,a3536e394b3f +-22.629,201,5.5,27de23496b0f +-22.519,201,6.7,27de23496b0f +-22.411,201,7.1,a3536e394b3f +-22.296,201,6.2,27de23496b0f +-22.187,201,6.8,27de23496b0f +-22.074,201,4.9,27de23496b0f +-21.968,201,5.4,a3536e394b3f +-21.859,201,5.4,27de23496b0f +-21.748,201,7.0,27de23496b0f +-21.638,201,7.7,27de23496b0f +-21.523,201,5.6,27de23496b0f +-21.41,201,9.1,a3536e394b3f +-21.295,201,6.5,a3536e394b3f +-21.18,201,7.0,a3536e394b3f +-21.063,201,6.9,27de23496b0f +-20.946,201,6.4,a3536e394b3f +-20.836,201,6.8,a3536e394b3f +-20.724,201,7.1,a3536e394b3f +-20.614,201,5.1,a3536e394b3f +-20.507,201,4.9,a3536e394b3f +-20.392,201,6.6,a3536e394b3f +-20.277,201,7.6,27de23496b0f +-20.161,201,7.8,a3536e394b3f +-20.047,201,4.9,a3536e394b3f +-19.94,201,8.0,27de23496b0f +-19.822,201,7.7,a3536e394b3f +-19.711,201,5.8,a3536e394b3f +-19.602,201,6.2,a3536e394b3f +-19.488,201,7.8,27de23496b0f +-19.38,201,7.7,27de23496b0f +-19.268,201,5.7,27de23496b0f +-19.154,201,6.3,27de23496b0f +-19.044,201,6.8,27de23496b0f +-18.93,201,6.2,27de23496b0f +-18.819,201,7.5,a3536e394b3f +-18.702,201,8.2,a3536e394b3f +-18.586,201,7.2,27de23496b0f +-18.475,201,6.7,27de23496b0f +-18.36,201,5.6,27de23496b0f +-18.253,201,7.1,27de23496b0f +-18.14,201,7.5,a3536e394b3f +-18.024,201,6.0,a3536e394b3f +-17.912,201,6.5,a3536e394b3f +-17.799,201,6.8,27de23496b0f +-17.686,201,6.2,27de23496b0f +-17.578,201,4.4,27de23496b0f +-17.473,201,4.0,a3536e394b3f +-17.367,201,4.3,a3536e394b3f +-17.262,201,3.8,27de23496b0f +-17.152,201,5.8,27de23496b0f +-17.038,201,8.1,a3536e394b3f +-16.928,201,7.2,a3536e394b3f +-16.81,201,5.3,a3536e394b3f +-16.702,201,4.6,a3536e394b3f +-16.594,201,6.0,a3536e394b3f +-16.488,201,6.7,a3536e394b3f +-16.373,201,7.9,a3536e394b3f +-16.257,201,6.6,27de23496b0f +-16.144,201,6.9,27de23496b0f +-16.031,201,5.8,27de23496b0f +-15.917,201,6.8,a3536e394b3f +-15.8,201,5.3,a3536e394b3f +-15.686,201,5.1,27de23496b0f +-15.58,201,5.9,a3536e394b3f +-15.469,201,7.7,27de23496b0f +-15.356,201,6.4,27de23496b0f +-15.249,201,7.1,a3536e394b3f +-15.138,201,7.1,27de23496b0f +-15.021,201,6.8,27de23496b0f +-14.911,201,6.3,a3536e394b3f +-14.798,201,16.4,27de23496b0f +-14.671,201,6.6,a3536e394b3f +-14.56,201,5.8,a3536e394b3f +-14.444,201,5.9,27de23496b0f +-14.329,201,7.3,a3536e394b3f +-14.212,201,7.1,a3536e394b3f +-14.102,201,7.0,a3536e394b3f +-13.986,201,5.4,27de23496b0f +-13.877,201,5.6,27de23496b0f +-13.764,201,6.7,27de23496b0f +-13.656,201,5.0,27de23496b0f +-13.544,201,4.9,27de23496b0f +-13.429,201,7.4,a3536e394b3f +-13.315,201,7.5,a3536e394b3f +-13.197,201,8.4,27de23496b0f +-13.086,201,7.7,27de23496b0f +-12.968,201,7.9,a3536e394b3f +-12.853,201,6.0,a3536e394b3f +-12.744,201,6.3,a3536e394b3f +-12.628,201,5.2,27de23496b0f +-12.513,201,4.7,27de23496b0f +-12.399,201,5.3,a3536e394b3f +-12.29,201,5.8,27de23496b0f +-12.183,201,6.2,27de23496b0f +-12.07,201,7.3,a3536e394b3f +-11.961,201,7.8,27de23496b0f +-11.847,201,5.7,a3536e394b3f +-11.739,201,8.0,a3536e394b3f +-11.622,201,3.8,a3536e394b3f +-11.512,201,6.9,a3536e394b3f +-11.402,201,5.1,a3536e394b3f +-11.292,201,5.8,a3536e394b3f +-11.185,201,6.4,a3536e394b3f +-11.069,201,7.0,27de23496b0f +-10.952,201,7.1,27de23496b0f +-10.837,201,8.2,a3536e394b3f +-10.719,201,7.9,27de23496b0f +-10.602,201,6.4,27de23496b0f +-10.494,201,7.2,27de23496b0f +-10.377,201,6.0,a3536e394b3f +-10.264,201,5.3,a3536e394b3f +-10.152,201,7.4,a3536e394b3f +-10.036,201,6.4,27de23496b0f +-9.919,201,7.4,27de23496b0f +-9.81,201,6.2,27de23496b0f +-9.696,201,7.1,a3536e394b3f +-9.586,201,5.1,a3536e394b3f +-9.475,201,5.8,a3536e394b3f +-9.36,201,6.8,a3536e394b3f +-9.248,201,5.8,27de23496b0f +-9.14,201,5.6,a3536e394b3f +-9.027,201,7.1,27de23496b0f +-8.92,201,7.1,a3536e394b3f +-8.803,201,7.2,27de23496b0f +-8.686,201,8.1,a3536e394b3f +-8.573,201,6.5,a3536e394b3f +-8.46,201,7.2,a3536e394b3f +-8.343,201,7.9,27de23496b0f +-8.227,201,7.4,a3536e394b3f +-8.119,201,9.7,a3536e394b3f +-8.005,201,8.0,a3536e394b3f +-7.894,201,5.1,a3536e394b3f +-7.779,201,5.8,27de23496b0f +-7.663,201,6.4,27de23496b0f +-7.549,201,5.0,27de23496b0f +-7.436,201,4.7,a3536e394b3f +-7.329,201,4.5,a3536e394b3f +-7.221,201,5.1,27de23496b0f +-7.107,201,6.9,a3536e394b3f +-6.994,201,7.0,a3536e394b3f +-6.877,201,7.2,27de23496b0f +-6.763,201,6.9,27de23496b0f +-6.646,201,6.9,a3536e394b3f +-6.529,201,7.1,27de23496b0f +-6.421,201,4.2,a3536e394b3f +-6.311,201,4.6,a3536e394b3f +-6.196,201,5.3,27de23496b0f +-6.081,201,5.8,27de23496b0f +-5.968,201,7.7,a3536e394b3f +-5.851,201,6.7,a3536e394b3f +-5.736,201,5.5,27de23496b0f +-5.624,201,6.8,27de23496b0f +-5.513,201,6.9,27de23496b0f +-5.4,201,6.5,27de23496b0f +-5.286,201,7.6,a3536e394b3f +-5.168,201,6.2,27de23496b0f +-5.055,201,6.8,27de23496b0f +-4.939,201,7.6,a3536e394b3f +-4.822,201,7.1,a3536e394b3f +-4.711,201,9.3,27de23496b0f +-4.595,201,6.2,27de23496b0f +-4.486,201,7.8,27de23496b0f +-4.377,201,6.0,27de23496b0f +-4.269,201,8.6,27de23496b0f +-4.152,201,6.7,27de23496b0f +-4.044,201,6.6,27de23496b0f +-3.927,201,5.7,a3536e394b3f +-3.819,201,10.0,a3536e394b3f +-3.701,201,7.4,a3536e394b3f +-3.586,201,6.0,27de23496b0f +-3.477,201,6.7,27de23496b0f +-3.36,201,4.5,27de23496b0f +-3.254,201,3.3,27de23496b0f +-3.144,201,5.9,27de23496b0f +-3.028,201,6.3,a3536e394b3f +-2.919,201,6.7,a3536e394b3f +-2.808,201,6.9,a3536e394b3f +-2.693,201,6.6,a3536e394b3f +-2.587,201,7.4,a3536e394b3f +-2.477,201,6.2,a3536e394b3f +-2.363,201,6.8,a3536e394b3f +-2.25,201,4.9,a3536e394b3f +-2.138,201,7.6,27de23496b0f +-2.03,201,7.6,27de23496b0f +-1.912,201,5.8,27de23496b0f +-1.797,201,6.8,a3536e394b3f +-1.689,201,4.7,27de23496b0f +-1.578,201,5.8,a3536e394b3f +-1.469,201,9.1,a3536e394b3f +-1.354,201,5.8,a3536e394b3f +-1.24,201,3.9,a3536e394b3f +-1.13,201,4.8,a3536e394b3f +-1.019,201,5.3,27de23496b0f +-0.904,201,6.3,27de23496b0f +-0.793,201,6.3,27de23496b0f +-0.678,201,7.0,a3536e394b3f +-0.565,201,5.9,a3536e394b3f +-0.455,201,7.7,a3536e394b3f +-0.34,201,8.3,27de23496b0f +-0.223,201,6.6,27de23496b0f +-0.114,201,6.8,27de23496b0f +-0.0,201,10.9,a3536e394b3f +0.12,201,4.4,a3536e394b3f +0.235,201,5.0,27de23496b0f +0.345,201,6.3,27de23496b0f +0.452,201,6.5,27de23496b0f +0.559,201,6.5,a3536e394b3f +0.667,201,6.5,a3536e394b3f +0.776,201,6.5,a3536e394b3f +0.883,201,8.3,a3536e394b3f +1.0,201,7.1,a3536e394b3f +1.11,201,6.7,a3536e394b3f +1.223,201,6.8,27de23496b0f +1.34,201,7.6,27de23496b0f +1.457,201,6.4,a3536e394b3f +1.572,201,23.0,27de23496b0f +1.705,201,6.6,27de23496b0f +1.819,201,7.4,27de23496b0f +1.936,201,6.6,27de23496b0f +2.048,201,7.8,27de23496b0f +2.161,201,7.2,27de23496b0f +2.275,201,5.3,27de23496b0f +2.388,201,7.4,27de23496b0f +2.498,201,6.4,27de23496b0f +2.612,201,5.0,27de23496b0f +2.725,201,4.8,27de23496b0f +2.839,201,5.6,27de23496b0f +2.948,201,7.2,27de23496b0f +3.056,201,5.4,27de23496b0f +3.166,201,8.2,27de23496b0f +3.282,201,10.8,27de23496b0f +3.397,201,7.0,27de23496b0f +3.505,201,7.0,27de23496b0f +3.622,201,8.1,27de23496b0f +3.731,201,9.8,27de23496b0f +3.846,201,4.9,27de23496b0f +3.961,201,6.5,27de23496b0f +4.078,201,5.5,27de23496b0f +4.189,201,6.7,27de23496b0f +4.306,201,5.7,27de23496b0f +4.418,201,7.7,27de23496b0f +4.528,201,4.9,27de23496b0f +4.642,201,9.1,27de23496b0f +4.757,201,5.8,27de23496b0f +4.866,201,6.9,27de23496b0f +4.978,201,8.8,27de23496b0f +5.096,201,5.8,27de23496b0f +5.206,201,9.3,27de23496b0f +5.316,201,26.2,27de23496b0f +5.449,201,5.7,27de23496b0f +5.563,201,7.5,27de23496b0f +5.678,201,6.7,27de23496b0f +5.795,201,6.7,27de23496b0f +5.908,201,7.6,27de23496b0f +6.023,201,6.0,27de23496b0f +6.129,201,7.1,27de23496b0f +6.237,201,6.5,27de23496b0f +6.348,201,5.3,27de23496b0f +6.458,201,3.9,27de23496b0f +6.572,201,7.5,27de23496b0f +6.681,201,7.0,27de23496b0f +6.789,201,5.3,27de23496b0f +6.904,201,6.2,27de23496b0f +7.019,201,7.5,27de23496b0f +7.127,201,9.2,27de23496b0f +7.242,201,8.2,27de23496b0f +7.351,201,6.9,27de23496b0f +7.46,201,6.9,27de23496b0f +7.572,201,7.8,27de23496b0f +7.69,201,7.8,27de23496b0f +7.804,201,6.8,27de23496b0f +7.917,201,6.5,27de23496b0f +8.031,201,7.7,27de23496b0f +8.147,201,7.0,27de23496b0f +8.26,201,8.5,27de23496b0f +8.378,201,6.5,27de23496b0f +8.489,201,7.2,27de23496b0f +8.601,201,7.2,27de23496b0f +8.718,201,6.9,27de23496b0f +8.828,201,6.3,27de23496b0f +8.94,201,6.5,27de23496b0f +9.048,201,6.1,27de23496b0f +9.16,201,4.9,27de23496b0f +9.275,201,5.2,27de23496b0f +9.381,201,8.3,27de23496b0f +9.489,201,5.7,27de23496b0f +9.6,201,6.7,27de23496b0f +9.717,201,7.4,27de23496b0f +9.834,201,7.2,27de23496b0f +9.947,201,6.5,27de23496b0f +10.063,201,8.9,27de23496b0f +10.174,201,8.3,27de23496b0f +10.289,201,6.9,27de23496b0f +10.4,201,8.2,27de23496b0f +10.515,201,11.5,27de23496b0f +10.636,201,5.2,27de23496b0f +10.742,201,7.5,27de23496b0f +10.858,201,6.0,27de23496b0f +10.974,201,7.0,27de23496b0f +11.081,201,8.1,27de23496b0f +11.198,201,7.8,27de23496b0f +11.306,201,9.5,27de23496b0f +11.424,201,6.6,27de23496b0f +11.531,201,9.6,27de23496b0f +11.651,201,5.9,27de23496b0f +11.765,201,5.7,27de23496b0f +11.872,201,7.0,27de23496b0f +11.988,201,7.3,27de23496b0f +12.097,201,7.6,27de23496b0f +12.215,201,7.8,27de23496b0f +12.333,201,4.4,27de23496b0f +12.439,201,5.9,27de23496b0f +12.546,201,5.9,27de23496b0f +12.661,201,5.7,27de23496b0f +12.768,201,4.4,27de23496b0f +12.876,201,3.2,27de23496b0f +12.981,201,5.5,27de23496b0f +13.096,201,4.3,27de23496b0f +13.21,201,7.1,27de23496b0f +13.323,201,5.7,27de23496b0f +13.434,201,7.3,27de23496b0f +13.548,201,5.7,27de23496b0f +13.66,201,8.9,27de23496b0f +13.779,201,5.7,27de23496b0f +13.887,201,4.4,27de23496b0f +13.992,201,5.1,27de23496b0f +14.101,201,5.9,27de23496b0f +14.209,201,5.4,a3536e394b3f +14.323,201,4.8,27de23496b0f +14.437,201,3.5,27de23496b0f +14.547,201,5.4,27de23496b0f +14.66,201,5.9,27de23496b0f +14.768,201,4.8,27de23496b0f +14.876,201,5.4,27de23496b0f +14.989,201,8.1,a3536e394b3f +15.107,201,6.9,a3536e394b3f +15.214,201,8.6,a3536e394b3f +15.327,201,7.3,a3536e394b3f +15.434,201,7.5,a3536e394b3f +15.55,201,6.2,a3536e394b3f +15.664,201,7.7,27de23496b0f +15.782,201,10.5,a3536e394b3f +15.893,201,7.6,a3536e394b3f +16.005,201,6.7,a3536e394b3f +16.121,201,7.4,27de23496b0f +16.235,201,7.9,27de23496b0f +16.344,201,8.1,27de23496b0f +16.455,201,6.5,27de23496b0f +16.562,201,8.0,27de23496b0f +16.671,201,4.8,a3536e394b3f +16.786,201,5.9,27de23496b0f +16.896,201,6.8,27de23496b0f +17.012,201,5.8,a3536e394b3f +17.122,201,6.1,a3536e394b3f +17.236,201,6.2,a3536e394b3f +17.343,201,7.1,a3536e394b3f +17.456,201,8.1,a3536e394b3f +17.573,201,7.2,27de23496b0f +17.684,201,7.1,27de23496b0f +17.798,201,6.6,27de23496b0f +17.908,201,7.1,27de23496b0f +18.019,201,7.5,27de23496b0f +18.128,201,6.9,27de23496b0f +18.242,201,7.9,a3536e394b3f +18.351,201,8.2,a3536e394b3f +18.463,201,8.2,a3536e394b3f +18.572,201,6.3,a3536e394b3f +18.682,201,5.8,27de23496b0f +18.791,201,6.2,27de23496b0f +18.898,201,7.3,27de23496b0f +19.009,201,5.2,27de23496b0f +19.122,201,3.9,a3536e394b3f +19.227,201,4.6,27de23496b0f +19.341,201,4.5,27de23496b0f +19.455,201,4.4,a3536e394b3f +19.563,201,5.8,a3536e394b3f +19.67,201,4.4,27de23496b0f +19.782,201,6.6,a3536e394b3f +19.898,201,7.1,a3536e394b3f +20.014,201,6.8,27de23496b0f +20.126,201,5.1,27de23496b0f +20.234,201,4.6,27de23496b0f +20.348,201,8.9,27de23496b0f +20.466,201,6.6,a3536e394b3f +20.573,201,6.9,a3536e394b3f +20.687,201,6.3,a3536e394b3f +20.797,201,4.9,a3536e394b3f +20.906,201,5.2,a3536e394b3f +21.014,201,5.0,a3536e394b3f +21.128,201,5.2,a3536e394b3f +21.24,201,7.5,a3536e394b3f +21.348,201,8.9,a3536e394b3f +21.46,201,9.1,27de23496b0f +21.571,201,7.4,27de23496b0f +21.681,201,8.1,27de23496b0f +21.792,201,7.2,27de23496b0f +21.901,201,7.3,27de23496b0f +22.019,201,8.0,27de23496b0f +22.128,201,8.6,27de23496b0f +22.245,201,9.6,a3536e394b3f +22.362,201,6.1,27de23496b0f +22.477,201,7.0,27de23496b0f +22.594,201,6.9,a3536e394b3f +22.707,201,6.5,a3536e394b3f +22.818,201,5.4,a3536e394b3f +22.934,201,5.2,27de23496b0f +23.047,201,4.9,27de23496b0f +23.161,201,6.8,a3536e394b3f +23.273,201,7.4,a3536e394b3f +23.39,201,6.3,a3536e394b3f +23.497,201,5.6,a3536e394b3f +23.608,201,6.4,a3536e394b3f +23.723,201,6.6,27de23496b0f +23.831,201,7.2,27de23496b0f +23.944,201,7.9,27de23496b0f +24.061,201,7.1,a3536e394b3f +24.173,201,5.2,a3536e394b3f +24.278,201,4.3,27de23496b0f +24.393,201,4.5,a3536e394b3f +24.507,201,4.3,a3536e394b3f +24.62,201,4.7,a3536e394b3f +24.726,201,5.1,a3536e394b3f +24.841,201,6.5,27de23496b0f +24.957,201,5.3,27de23496b0f +25.063,201,7.6,27de23496b0f +25.178,201,5.9,27de23496b0f +25.289,201,6.4,27de23496b0f +25.406,201,6.4,a3536e394b3f +25.514,201,8.4,a3536e394b3f +25.631,201,8.2,27de23496b0f +25.748,201,6.8,27de23496b0f +25.861,201,7.8,a3536e394b3f +25.974,201,8.1,a3536e394b3f +26.089,201,8.4,27de23496b0f +26.206,201,6.3,27de23496b0f +26.314,201,6.7,27de23496b0f +26.422,201,7.6,27de23496b0f +26.535,201,6.9,27de23496b0f +26.65,201,7.6,a3536e394b3f +26.76,201,7.3,27de23496b0f +26.869,201,7.8,27de23496b0f +26.978,201,7.3,27de23496b0f +27.088,201,6.5,27de23496b0f +27.195,201,6.7,27de23496b0f +27.306,201,7.1,27de23496b0f +27.422,201,8.3,a3536e394b3f +27.531,201,8.0,a3536e394b3f +27.647,201,7.6,a3536e394b3f +27.761,201,7.6,27de23496b0f +27.879,201,7.7,27de23496b0f +27.993,201,7.2,a3536e394b3f +28.108,201,6.6,a3536e394b3f +28.223,201,8.5,27de23496b0f +28.34,201,8.0,a3536e394b3f +28.456,201,6.5,a3536e394b3f +28.573,201,8.0,27de23496b0f +28.681,201,7.9,27de23496b0f +28.797,201,6.6,27de23496b0f +28.906,201,6.2,27de23496b0f +29.013,201,6.2,27de23496b0f +29.127,201,6.4,27de23496b0f +29.237,201,6.2,27de23496b0f +29.346,201,5.2,27de23496b0f +29.458,201,6.6,27de23496b0f +29.568,201,6.0,27de23496b0f +29.681,201,4.6,a3536e394b3f +29.792,201,5.5,a3536e394b3f +29.907,201,6.3,a3536e394b3f +30.019,201,8.3,a3536e394b3f +30.129,201,6.1,a3536e394b3f +30.236,201,7.8,a3536e394b3f +30.348,201,9.3,a3536e394b3f +30.458,201,9.5,a3536e394b3f +30.57,201,9.0,27de23496b0f +30.681,201,8.6,27de23496b0f +30.794,201,8.0,27de23496b0f +30.912,201,7.5,a3536e394b3f +31.022,201,8.2,a3536e394b3f +31.131,201,8.2,a3536e394b3f +31.248,201,6.7,a3536e394b3f +31.36,201,8.0,27de23496b0f +31.472,201,6.3,27de23496b0f +31.579,201,9.8,a3536e394b3f +31.691,201,7.2,27de23496b0f +31.806,201,7.8,27de23496b0f +31.915,201,8.9,27de23496b0f +32.031,201,7.6,a3536e394b3f +32.147,201,6.5,a3536e394b3f +32.256,201,7.6,a3536e394b3f +32.374,201,8.6,27de23496b0f +32.486,201,8.4,27de23496b0f +32.598,201,9.3,27de23496b0f +32.707,201,6.7,27de23496b0f +32.824,201,7.5,a3536e394b3f +32.942,201,6.2,27de23496b0f +33.048,201,6.3,a3536e394b3f +33.161,201,4.7,a3536e394b3f +33.276,201,4.9,27de23496b0f +33.386,201,5.4,27de23496b0f +33.496,201,5.5,27de23496b0f +33.61,201,5.7,27de23496b0f +33.716,201,4.6,27de23496b0f +33.826,201,5.6,27de23496b0f +33.935,201,5.5,27de23496b0f +34.047,201,6.1,27de23496b0f +34.156,201,6.0,27de23496b0f +34.265,201,7.2,27de23496b0f +34.378,201,6.6,27de23496b0f +34.49,201,5.1,27de23496b0f +34.6,201,4.4,27de23496b0f +34.712,201,4.7,a3536e394b3f +34.818,201,3.1,27de23496b0f +34.928,201,5.7,27de23496b0f +35.037,201,5.9,27de23496b0f +35.148,201,6.6,27de23496b0f +35.258,201,4.9,27de23496b0f +35.373,201,5.3,a3536e394b3f +35.479,201,5.1,27de23496b0f +35.594,201,7.7,a3536e394b3f +35.702,201,6.6,a3536e394b3f +35.815,201,6.5,a3536e394b3f +35.923,201,8.9,a3536e394b3f +36.037,201,4.8,a3536e394b3f +36.15,201,4.6,a3536e394b3f +36.26,201,5.3,a3536e394b3f +36.373,201,6.0,a3536e394b3f +36.486,201,6.2,27de23496b0f +36.603,201,4.7,a3536e394b3f +36.708,201,5.8,27de23496b0f +36.816,201,5.8,27de23496b0f +36.925,201,6.3,27de23496b0f +37.042,201,8.9,a3536e394b3f +37.16,201,7.6,27de23496b0f +37.277,201,6.2,27de23496b0f +37.389,201,5.0,27de23496b0f +37.495,201,5.7,27de23496b0f +37.608,201,7.4,27de23496b0f +37.723,201,7.3,a3536e394b3f +37.835,201,7.6,a3536e394b3f +37.951,201,7.9,27de23496b0f +38.065,201,6.6,27de23496b0f +38.179,201,6.9,27de23496b0f +38.291,201,7.0,a3536e394b3f +38.408,201,7.0,a3536e394b3f +38.521,201,7.2,27de23496b0f +38.634,201,7.8,27de23496b0f +38.743,201,8.1,27de23496b0f +38.861,201,8.3,a3536e394b3f +38.977,201,6.9,a3536e394b3f +39.089,201,7.4,a3536e394b3f +39.2,201,5.2,a3536e394b3f +39.314,201,7.1,27de23496b0f +39.423,201,7.1,27de23496b0f +39.54,201,7.3,a3536e394b3f +39.656,201,5.4,a3536e394b3f +39.762,201,5.1,a3536e394b3f +39.874,201,4.3,a3536e394b3f +39.986,201,5.7,a3536e394b3f +40.094,201,6.1,a3536e394b3f +40.206,201,4.8,a3536e394b3f +40.321,201,7.2,27de23496b0f +40.438,201,7.2,27de23496b0f +40.555,201,8.0,a3536e394b3f +40.669,201,5.9,a3536e394b3f +40.783,201,6.4,27de23496b0f +40.899,201,9.0,a3536e394b3f +41.018,201,12.7,27de23496b0f +41.14,201,5.0,27de23496b0f +41.247,201,4.8,27de23496b0f +41.361,201,8.2,a3536e394b3f +41.476,201,6.7,a3536e394b3f +41.59,201,6.6,a3536e394b3f +41.706,201,6.2,27de23496b0f +41.823,201,7.8,a3536e394b3f +41.932,201,7.7,a3536e394b3f +42.045,201,6.1,a3536e394b3f +42.16,201,7.0,27de23496b0f +42.277,201,7.4,27de23496b0f +42.39,201,6.9,27de23496b0f +42.498,201,5.9,27de23496b0f +42.611,201,7.6,a3536e394b3f +42.723,201,7.2,a3536e394b3f +42.834,201,7.9,a3536e394b3f +42.952,201,7.8,27de23496b0f +43.06,201,7.7,a3536e394b3f +43.171,201,7.0,a3536e394b3f +43.278,201,6.3,a3536e394b3f +43.394,201,12.4,27de23496b0f +43.515,201,6.2,a3536e394b3f +43.627,201,6.2,a3536e394b3f +43.744,201,7.5,27de23496b0f +43.86,201,7.9,a3536e394b3f +43.973,201,7.4,a3536e394b3f +44.088,201,7.9,a3536e394b3f +44.199,201,7.8,a3536e394b3f +44.315,201,8.5,27de23496b0f +44.424,201,6.6,27de23496b0f +44.541,201,7.7,a3536e394b3f +44.658,201,7.5,a3536e394b3f +44.771,201,7.9,27de23496b0f +44.88,201,6.1,a3536e394b3f +44.996,201,5.1,27de23496b0f +45.109,201,5.6,a3536e394b3f +45.221,201,5.4,27de23496b0f +45.333,201,7.0,a3536e394b3f +45.44,201,6.2,27de23496b0f +45.549,201,5.9,a3536e394b3f diff --git a/evidence/failover-each/a3536e394b3f/result.json b/evidence/failover-each/a3536e394b3f/result.json new file mode 100644 index 00000000..79605448 --- /dev/null +++ b/evidence/failover-each/a3536e394b3f/result.json @@ -0,0 +1,60 @@ +{ + "label": "a3536e394b3f", + "action": [ + "sh", + "-c", + "scripts/stop-one.sh && sleep 10 && scripts/restore.sh" + ], + "action_exit": 0, + "action_seconds": 14.511, + "action_finished_before_load_end": true, + "lb_up_before": 2, + "lb_up_min": 1, + "detection_seconds": 1.515, + "rejoin_seconds": 14.111, + "probe": { + "attempts": 635, + "failed_attempts": 0, + "first_failure_s": null, + "recovery_s": 0.0, + "instances_before": { + "a3536e394b3f": 123, + "27de23496b0f": 107 + }, + "instances_during_outage": { + "27de23496b0f": 111 + }, + "instances_after": { + "27de23496b0f": 144, + "a3536e394b3f": 133 + } + }, + "lb_counters": { + "wretr": 0, + "wredis": 0, + "econ": 0, + "eresp": 0, + "hrsp_5xx": 0 + }, + "k6": { + "requests": 6001, + "rps": 100.02, + "ok_rps": 100.02, + "failed_ratio": 0, + "dropped_iterations": 0, + "p50_ms": 2.4, + "p95_ms": 3.41, + "p99_ms": 5.1, + "exit": 0, + "rate": 100 + }, + "checks": { + "action_ok": true, + "load_errors_ok": true, + "write_recovery_ok": true, + "detected": true, + "target_excluded": true, + "rejoined": true + }, + "pass": true +} diff --git a/evidence/failover-each/summary.json b/evidence/failover-each/summary.json new file mode 100644 index 00000000..618524fc --- /dev/null +++ b/evidence/failover-each/summary.json @@ -0,0 +1,122 @@ +[ + { + "label": "a3536e394b3f", + "action": [ + "sh", + "-c", + "scripts/stop-one.sh && sleep 10 && scripts/restore.sh" + ], + "action_exit": 0, + "action_seconds": 14.511, + "action_finished_before_load_end": true, + "lb_up_before": 2, + "lb_up_min": 1, + "detection_seconds": 1.515, + "rejoin_seconds": 14.111, + "probe": { + "attempts": 635, + "failed_attempts": 0, + "first_failure_s": null, + "recovery_s": 0.0, + "instances_before": { + "a3536e394b3f": 123, + "27de23496b0f": 107 + }, + "instances_during_outage": { + "27de23496b0f": 111 + }, + "instances_after": { + "27de23496b0f": 144, + "a3536e394b3f": 133 + } + }, + "lb_counters": { + "wretr": 0, + "wredis": 0, + "econ": 0, + "eresp": 0, + "hrsp_5xx": 0 + }, + "k6": { + "requests": 6001, + "rps": 100.02, + "ok_rps": 100.02, + "failed_ratio": 0, + "dropped_iterations": 0, + "p50_ms": 2.4, + "p95_ms": 3.41, + "p99_ms": 5.1, + "exit": 0, + "rate": 100 + }, + "checks": { + "action_ok": true, + "load_errors_ok": true, + "write_recovery_ok": true, + "detected": true, + "target_excluded": true, + "rejoined": true + }, + "pass": true + }, + { + "label": "27de23496b0f", + "action": [ + "sh", + "-c", + "scripts/stop-one.sh && sleep 10 && scripts/restore.sh" + ], + "action_exit": 0, + "action_seconds": 14.508, + "action_finished_before_load_end": true, + "lb_up_before": 2, + "lb_up_min": 1, + "detection_seconds": 0.894, + "rejoin_seconds": 14.392, + "probe": { + "attempts": 635, + "failed_attempts": 0, + "first_failure_s": null, + "recovery_s": 0.0, + "instances_before": { + "a3536e394b3f": 118, + "27de23496b0f": 111 + }, + "instances_during_outage": { + "a3536e394b3f": 119 + }, + "instances_after": { + "a3536e394b3f": 152, + "27de23496b0f": 124 + } + }, + "lb_counters": { + "wretr": 0, + "wredis": 0, + "econ": 0, + "eresp": 0, + "hrsp_5xx": 0 + }, + "k6": { + "requests": 6001, + "rps": 100.02, + "ok_rps": 100.02, + "failed_ratio": 0, + "dropped_iterations": 0, + "p50_ms": 2.44, + "p95_ms": 3.42, + "p99_ms": 5.08, + "exit": 0, + "rate": 100 + }, + "checks": { + "action_ok": true, + "load_errors_ok": true, + "write_recovery_ok": true, + "detected": true, + "target_excluded": true, + "rejoined": true + }, + "pass": true + } +] diff --git a/evidence/faults.txt b/evidence/faults.txt index 0252fce1..284198c5 100644 --- a/evidence/faults.txt +++ b/evidence/faults.txt @@ -1,7 +1,12 @@ -date: 2026-10-28T08:49:44Z commit: 0796e29 -$ make test-faults BASE_URL=http://localhost:18083 # тесты стенда лабы 2 через балансировщик +date: 2026-10-28T16:57:24Z commit: e00eb6c +$ make test-faults BASE_URL=http://localhost:18083 BASE_URL="http://localhost:18083" .venv/bin/pytest -m stack -v -s student-tests/stack ============================= test session starts ============================== +platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 +rootdir: /Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab3 +configfile: pyproject.toml +plugins: asyncio-1.4.0, anyio-4.15.1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function collected 10 items student-tests/stack/test_faults.py гонка 50 регистраций: 201×1, 409×49 @@ -9,23 +14,24 @@ student-tests/stack/test_faults.py гонка 50 регистраций: 201×1, .неверный пароль: 401 за 0.012 с неизвестный пользователь: 401 за 0.012 с .logout: 204, затем 401, повтор 401 -.5 проверок токена: keyspace_hits 394 → 399 +.5 проверок токена: keyspace_hits 4 → 9 .valkey остановлен, действующий токен: 200 за 0.004 с valkey остановлен, неизвестный токен: 401 за 0.002 с valkey остановлен: ready 200, вход 201, выход 503 valkey снова запущен: повторный выход 204 -.рестарт valkey: ключей 28 → 28 +.рестарт valkey: ключей 25 → 25 рестарт valkey: живой 200, отозванный 401 .FLUSHALL: живой 200, отозванный 401 -.postgres остановлен, ready: 503 за 0.003 с -postgres остановлен, live: 200 за 0.001 с -postgres остановлен, me: 503 за 0.002 с -postgres остановлен, stranger: 503 за 0.003 с +.postgres остановлен, ready: 503 за 0.004 с +postgres остановлен, me: 503 за 0.003 с +postgres остановлен, stranger: 503 за 0.002 с postgres остановлен, login: 503 за 0.002 с postgres остановлен, register: 503 за 0.014 с -postgres запущен: готовность через 0.3 с, прежний токен 200 -.рестарт api: готовность через 0.5 с, токен 200 -expires_at при входе 2026-10-28T09:49:55.730592Z, в БД после рестарта 2026-10-28T09:49:55.730592Z +live внутри реплики 200; LB исключил все реплики ещё через 0.61 с +live через LB без реплик: 503 за 0.000 с +postgres запущен: готовность через 2.1 с, прежний токен 200 +.рестарт api: готовность через 2.8 с, токен 200 +expires_at при входе 2026-10-28T17:57:38.249277Z, в БД после рестарта 2026-10-28T17:57:38.249277Z . -============================= 10 passed in 11.99s ============================== +============================= 10 passed in 19.25s ============================== diff --git a/evidence/hooks.txt b/evidence/hooks.txt index 8ade70c0..ad448071 100644 --- a/evidence/hooks.txt +++ b/evidence/hooks.txt @@ -1,21 +1,21 @@ -date: 2026-10-28T08:48:13Z commit: 0796e29 -$ TARGET_INSTANCE_ID= scripts/stop-one.sh; echo exit=$? # seconds — длительность хука -target=302cfee8f673 exit=0 seconds=0.68 -302cfee8f673 exited exit_code=0 oom=false -b582bd503bdd running exit_code=0 oom=false +date: 2026-10-28T16:56:54Z commit: e00eb6c +$ TARGET_INSTANCE_ID= scripts/stop-one.sh; echo exit=$? # seconds — длительность хука (окно вывода из ротации 2 с + остановка) +target=a3536e394b3f exit=0 seconds=2.72 +a3536e394b3f exited exit_code=0 oom=false +27de23496b0f running exit_code=0 oom=false $ scripts/restore.sh exit=0 -302cfee8f673 running -b582bd503bdd running +a3536e394b3f running +27de23496b0f running $ TARGET_INSTANCE_ID=no-such-replica scripts/stop-one.sh; echo exit=$? no running api replica answers as no-such-replica exit=3 $ docker compose ps --format '{{.Service}} {{.Name}} {{.Status}}' # ничего не остановлено api mtusi2026lab3-api-1 Up 2 seconds (healthy) -api mtusi2026lab3-api-2 Up 13 seconds (healthy) -lb mtusi2026lab3-lb-1 Up About a minute (healthy) -postgres mtusi2026lab3-postgres-1 Up About a minute (healthy) -valkey mtusi2026lab3-valkey-1 Up About a minute (healthy) +api mtusi2026lab3-api-2 Up 16 seconds (healthy) +lb mtusi2026lab3-lb-1 Up 3 minutes (healthy) +postgres mtusi2026lab3-postgres-1 Up 3 minutes (healthy) +valkey mtusi2026lab3-valkey-1 Up 3 minutes (healthy) $ scripts/stop-one.sh # без TARGET_INSTANCE_ID scripts/stop-one.sh: line 3: TARGET_INSTANCE_ID: test provides target API instance alias exit=1 diff --git a/evidence/lb-check.json b/evidence/lb-check.json index 21b82950..d52d4919 100644 --- a/evidence/lb-check.json +++ b/evidence/lb-check.json @@ -1,20 +1,21 @@ { "instances_one_token": { - "b582bd503bdd": 20, - "302cfee8f673": 20 + "27de23496b0f": 20, + "a3536e394b3f": 20 }, "same_key": { + "rounds": 20, "parallel": 8, "statuses": { - "201": 8 + "201": 160 }, "instances": { - "b582bd503bdd": 4, - "302cfee8f673": 4 + "a3536e394b3f": 80, + "27de23496b0f": 80 }, - "identical_bodies": true, - "stored_copies": 1, - "other_body_status": 409 + "identical_bodies_rounds": 20, + "single_copy_rounds": 20, + "conflict_rounds": 20 }, "pass": true } diff --git a/evidence/limits-violations.txt b/evidence/limits-violations.txt new file mode 100644 index 00000000..2486d912 --- /dev/null +++ b/evidence/limits-violations.txt @@ -0,0 +1,20 @@ +date: 2026-10-28T17:07:12Z commit: e00eb6c +# намеренные нарушения политики во временном override (не в репозитории): +# services: +# api: +# ports: ["8081:8080"] +# secrets: [database_url, cache_url, cursor_secret, postgres_password] +# valkey: +# container_name: lab3-valkey +# k6: +# deploy: +# resources: +# limits: {cpus: "0", memory: "0"} +$ COMPOSE_FILE=compose.yaml:violations.yaml python3 scripts/limits.py; echo exit=$? # строки FAIL/PASS +FAIL: api: secrets ['cache_url', 'cursor_secret', 'database_url', 'postgres_password'], policy ['cache_url', 'cursor_secret', 'database_url'] +FAIL: api: publishes ['0.0.0.0:8081'] in bypass of the balancer +FAIL: api: publishes ['0.0.0.0:8081'] beyond loopback +FAIL: k6: no CPU or memory limit +FAIL: valkey: container_name prevents scaling +FAIL: 5 violation(s) +exit=1 diff --git a/evidence/limits.txt b/evidence/limits.txt new file mode 100644 index 00000000..2482efe6 --- /dev/null +++ b/evidence/limits.txt @@ -0,0 +1,21 @@ +date: 2026-10-28T16:53:57Z +$ docker compose --profile bench config --format json (declared) +service cpus/mem ports secrets volumes networks ro +-------- ---------- ------------------------------- ------------------------------------ ---------- ---------------------- ----- +api 1/512MiB - cache_url,cursor_secret,database_url - backend(internal) true +k6 2/512MiB - - - edge false +lb 1/128MiB 127.0.0.1:18083,127.0.0.1:23083 - - backend(internal),edge true +migrate 0.5/256MiB - database_url - backend(internal) false +postgres 2/1024MiB - postgres_password pgdata backend(internal) false +valkey 0.5/128MiB - valkey.conf valkeydata backend(internal) false + +$ docker inspect (running containers: hostname = X-Instance-Id, applied limits) +service hostname cpus memory published +-------- ------------ ---- ------- --------------------------------------------------- +api 27de23496b0f 1 512MiB +api a3536e394b3f 1 512MiB +lb 0b152ec25c09 1 128MiB 127.0.0.1:18083->8080/tcp,127.0.0.1:23083->8404/tcp +postgres 67fe9b43b70b 2 1024MiB +valkey e4c5a748222e 0.5 128MiB + +PASS: limits, ports, secrets, volumes and networks match the policy diff --git a/evidence/persistence.txt b/evidence/persistence.txt index f184e2ad..e7ba2a07 100644 --- a/evidence/persistence.txt +++ b/evidence/persistence.txt @@ -1,18 +1,18 @@ -date: 2026-10-28T08:47:06Z commit: 0796e29 +date: 2026-10-28T16:57:10Z commit: e00eb6c == before: service hostname StartedAt status -api 302cfee8f673 2026-10-28T08:46:57.096571506Z running -api b582bd503bdd 2026-10-28T08:46:57.018519589Z running -lb 10cd61cb6cc5 2026-10-28T08:46:50.904998045Z running -migrate d7aac30f1875 2026-10-28T08:46:55.919807755Z exited -postgres f06fc06e6824 2026-10-28T08:46:50.659210753Z running -valkey 5f8773e8a7ad 2026-10-28T08:46:50.704690336Z running +api 27de23496b0f 2026-10-28T16:57:02.736671133Z running +api a3536e394b3f 2026-10-28T16:57:02.799486467Z running +lb 0b152ec25c09 2026-10-28T16:53:41.547744012Z running +migrate a70426d17a7b 2026-10-28T16:53:47.172744001Z exited +postgres 67fe9b43b70b 2026-10-28T16:53:41.549730471Z running +valkey e4c5a748222e 2026-10-28T16:53:41.550427137Z running python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18083" Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. == after: service hostname StartedAt status -api 302cfee8f673 2026-10-28T08:47:14.781792917Z running -api b582bd503bdd 2026-10-28T08:47:14.713654417Z running -lb 10cd61cb6cc5 2026-10-28T08:47:08.587781136Z running -migrate d7aac30f1875 2026-10-28T08:47:13.605412347Z exited -postgres f06fc06e6824 2026-10-28T08:47:08.243716595Z running -valkey 5f8773e8a7ad 2026-10-28T08:47:08.381440386Z running +api 27de23496b0f 2026-10-28T16:57:19.87642046Z running +api a3536e394b3f 2026-10-28T16:57:19.954143294Z running +lb 0b152ec25c09 2026-10-28T16:57:13.78578743Z running +migrate a70426d17a7b 2026-10-28T16:57:18.79001746Z exited +postgres 67fe9b43b70b 2026-10-28T16:57:13.56121518Z running +valkey e4c5a748222e 2026-10-28T16:57:13.579042805Z running diff --git a/evidence/rolling-from-8d28d45/action.txt b/evidence/rolling-from-8d28d45/action.txt new file mode 100644 index 00000000..e234dee0 --- /dev/null +++ b/evidence/rolling-from-8d28d45/action.txt @@ -0,0 +1,29 @@ +17:01:24 migrate once before the first replacement +{"reason":"up to date","event":"migration.completed","level":"info","ts":"2026-10-28T17:01:24.982557Z"} + Container mtusi2026lab3-api-1 Running + Container mtusi2026lab3-api-2 Running + Container mtusi2026lab3-api-3 Creating + Container mtusi2026lab3-api-3 Created + Container mtusi2026lab3-api-3 Starting + Container mtusi2026lab3-api-3 Started + Container mtusi2026lab3-api-3 Waiting + Container mtusi2026lab3-api-2 Waiting + Container mtusi2026lab3-api-1 Waiting + Container mtusi2026lab3-api-3 Healthy + Container mtusi2026lab3-api-2 Healthy + Container mtusi2026lab3-api-1 Healthy +17:01:27 replace 7a4c320f9b13 image=cbd83541c3e5 -> 28af1eaa77c8 image=13e52dc26774 + Container mtusi2026lab3-api-2 Running + Container mtusi2026lab3-api-3 Running + Container mtusi2026lab3-api-4 Creating + Container mtusi2026lab3-api-4 Created + Container mtusi2026lab3-api-4 Starting + Container mtusi2026lab3-api-4 Started + Container mtusi2026lab3-api-4 Waiting + Container mtusi2026lab3-api-3 Waiting + Container mtusi2026lab3-api-2 Waiting + Container mtusi2026lab3-api-4 Healthy + Container mtusi2026lab3-api-3 Healthy + Container mtusi2026lab3-api-2 Healthy +17:01:31 replace 00548d9b229e image=cbd83541c3e5 -> f3f2f7801b56 image=13e52dc26774 +17:01:31 done: 2 replicas replaced diff --git a/evidence/rolling-from-8d28d45/k6/k6.txt b/evidence/rolling-from-8d28d45/k6/k6.txt new file mode 100644 index 00000000..ae6165bf --- /dev/null +++ b/evidence/rolling-from-8d28d45/k6/k6.txt @@ -0,0 +1,2 @@ +time="2026-10-28T17:00:59Z" level=info msg=LOAD-START source=console +measure: 9000 requests, failed ratio 0 diff --git a/evidence/rolling-from-8d28d45/k6/summary.json b/evidence/rolling-from-8d28d45/k6/summary.json new file mode 100644 index 00000000..e3093b69 --- /dev/null +++ b/evidence/rolling-from-8d28d45/k6/summary.json @@ -0,0 +1,309 @@ +{ + "metrics": { + "http_req_duration{scenario:measure}": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 3.1322913, + "p(95)": 3.4010535500000003, + "p(99)": 5.193993249999999, + "max": 505.477417, + "avg": 3.453344209333336, + "min": 1.0105, + "med": 2.4411465000000003 + } + }, + "iteration_duration": { + "values": { + "p(90)": 3.25575, + "p(95)": 3.549833, + "p(99)": 5.436542, + "max": 505.569667, + "avg": 3.4618407749225035, + "min": 1.011083, + "med": 2.544833 + }, + "type": "trend", + "contains": "time" + }, + "http_reqs{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 9000, + "rate": 89.56211577313219 + } + }, + "http_req_waiting": { + "contains": "time", + "values": { + "p(90)": 3.088625, + "p(95)": 3.4003954999999997, + "p(99)": 5.426020999999999, + "max": 505.430667, + "avg": 3.3401909016018263, + "min": 0.955208, + "med": 2.394167 + }, + "type": "trend" + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 2930946, + "rate": 29166.85833075541 + } + }, + "dropped_iterations{scenario:measure}": { + "values": { + "count": 1, + "rate": 0.009951346197014687 + }, + "type": "counter", + "contains": "default" + }, + "dropped_iterations": { + "contains": "default", + "values": { + "count": 1, + "rate": 0.009951346197014687 + }, + "type": "counter" + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 10001, + "rate": 99.52341331634389 + } + }, + "http_req_duration{scenario:measure,name:post_message}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 3.5837394056486134, + "min": 1.047208, + "med": 2.474209, + "p(90)": 3.1399334000000003, + "p(95)": 3.3924048, + "p(99)": 5.216426739999998, + "max": 505.477417 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 24696511, + "rate": 245763.5308193814 + } + }, + "vus_max": { + "contains": "default", + "values": { + "max": 21, + "value": 21, + "min": 20 + }, + "type": "gauge" + }, + "iterations{scenario:measure}": { + "values": { + "rate": 89.56211577313219, + "count": 9000 + }, + "type": "counter", + "contains": "default" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "p(99)": 0.07410399999999999, + "max": 0.44875, + "avg": 0.018128176997313703, + "min": 0.001958, + "med": 0.016209, + "p(90)": 0.021917, + "p(95)": 0.02972899999999998 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 10051, + "rate": 100.02098062619463 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "p(90)": 0, + "p(95)": 0, + "p(99)": 0, + "max": 0.300042, + "avg": 0.0003657015222365933 + } + }, + "http_req_duration{scenario:measure,name:get_history}": { + "contains": "time", + "values": { + "p(99)": 5.059531159999999, + "max": 504.407417, + "avg": 2.9233247929093955, + "min": 1.0105, + "med": 2.328833, + "p(90)": 3.0485334, + "p(95)": 3.4437255999999996 + }, + "type": "trend" + }, + "http_req_failed{scenario:measure}": { + "values": { + "passes": 0, + "fails": 9000, + "rate": 0 + }, + "type": "rate", + "contains": "default" + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "max": 2.285666, + "avg": 0.029886806486916724, + "min": 0.008875, + "med": 0.026833, + "p(90)": 0.041709, + "p(95)": 0.052542, + "p(99)": 0.09135399999999999 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "max": 505.477417, + "avg": 3.3882058850860473, + "min": 0.975791, + "med": 2.443, + "p(90)": 3.14625, + "p(95)": 3.465979, + "p(99)": 5.5215204999999985 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 1, + "min": 0, + "max": 5 + } + }, + "http_req_failed": { + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 10051 + }, + "type": "rate" + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 10001, + "fails": 0 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "p(90)": 0, + "p(95)": 0, + "p(99)": 0, + "max": 0 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "med": 0.001916, + "p(90)": 0.002833, + "p(95)": 0.0035, + "p(99)": 0.012811999999999997, + "max": 0.495583, + "avg": 0.002694060988956286, + "min": 0.000542 + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 505.477417, + "avg": 3.3882058850860473, + "min": 0.975791, + "med": 2.443, + "p(90)": 3.14625, + "p(95)": 3.465979, + "p(99)": 5.5215204999999985 + } + } + }, + "root_group": { + "checks": [ + { + "name": "message 201", + "path": "::message 201", + "id": "6445aff337e1ade0bc10402fbac79882", + "passes": 8024, + "fails": 0 + }, + { + "name": "history 200", + "path": "::history 200", + "id": "fe27b3b8edd292c7f2170a6357b087d0", + "passes": 1977, + "fails": 0 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "p(90)", + "p(95)", + "p(99)", + "max" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": false, + "isStdErrTTY": false, + "testRunDurationMs": 100488.916796 + } +} \ No newline at end of file diff --git a/evidence/rolling-from-8d28d45/lb_up.csv b/evidence/rolling-from-8d28d45/lb_up.csv new file mode 100644 index 00000000..8c0180e9 --- /dev/null +++ b/evidence/rolling-from-8d28d45/lb_up.csv @@ -0,0 +1,915 @@ +t_rel_s,up +-25.776,2 +-25.669,2 +-25.565,2 +-25.462,2 +-25.351,2 +-25.244,2 +-25.135,2 +-25.026,2 +-24.92,2 +-24.813,2 +-24.699,2 +-24.587,2 +-24.476,2 +-24.366,2 +-24.259,2 +-24.148,2 +-24.034,2 +-23.928,2 +-23.825,2 +-23.71,2 +-23.601,2 +-23.49,2 +-23.379,2 +-23.271,2 +-23.164,2 +-23.051,2 +-22.942,2 +-22.834,2 +-22.717,2 +-22.602,2 +-22.492,2 +-22.385,2 +-22.271,2 +-22.16,2 +-22.048,2 +-21.942,2 +-21.836,2 +-21.721,2 +-21.609,2 +-21.496,2 +-21.384,2 +-21.271,2 +-21.156,2 +-21.04,2 +-20.925,2 +-20.818,2 +-20.703,2 +-20.597,2 +-20.485,2 +-20.369,2 +-20.258,2 +-20.148,2 +-20.037,2 +-19.931,2 +-19.816,2 +-19.7,2 +-19.593,2 +-19.487,2 +-19.381,2 +-19.271,2 +-19.165,2 +-19.059,2 +-18.952,2 +-18.842,2 +-18.727,2 +-18.618,2 +-18.51,2 +-18.395,2 +-18.279,2 +-18.172,2 +-18.067,2 +-17.953,2 +-17.848,2 +-17.744,2 +-17.639,2 +-17.533,2 +-17.427,2 +-17.314,2 +-17.205,2 +-17.098,2 +-16.992,2 +-16.875,2 +-16.769,2 +-16.658,2 +-16.544,2 +-16.434,2 +-16.319,2 +-16.205,2 +-16.095,2 +-15.985,2 +-15.878,2 +-15.762,2 +-15.651,2 +-15.536,2 +-15.43,2 +-15.322,2 +-15.206,2 +-15.09,2 +-14.982,2 +-14.869,2 +-14.759,2 +-14.648,2 +-14.533,2 +-14.427,2 +-14.314,2 +-14.204,2 +-14.096,2 +-13.99,2 +-13.877,2 +-13.767,2 +-13.655,2 +-13.541,2 +-13.433,2 +-13.318,2 +-13.205,2 +-13.092,2 +-12.985,2 +-12.875,2 +-12.765,2 +-12.65,2 +-12.536,2 +-12.431,2 +-12.317,2 +-12.209,2 +-12.099,2 +-11.986,2 +-11.872,2 +-11.759,2 +-11.646,2 +-11.54,2 +-11.429,2 +-11.318,2 +-11.212,2 +-11.096,2 +-10.981,2 +-10.866,2 +-10.759,2 +-10.647,2 +-10.542,2 +-10.43,2 +-10.318,2 +-10.203,2 +-10.096,2 +-9.991,2 +-9.884,2 +-9.768,2 +-9.656,2 +-9.551,2 +-9.437,2 +-9.322,2 +-9.208,2 +-9.093,2 +-8.977,2 +-8.865,2 +-8.751,2 +-8.643,2 +-8.535,2 +-8.421,2 +-8.306,2 +-8.191,2 +-8.081,2 +-7.965,2 +-7.852,2 +-7.743,2 +-7.634,2 +-7.526,2 +-7.415,2 +-7.3,2 +-7.185,2 +-7.07,2 +-6.959,2 +-6.849,2 +-6.735,2 +-6.62,2 +-6.513,2 +-6.401,2 +-6.292,2 +-6.177,2 +-6.063,2 +-5.951,2 +-5.835,2 +-5.726,2 +-5.613,2 +-5.505,2 +-5.397,2 +-5.292,2 +-5.178,2 +-5.063,2 +-4.947,2 +-4.836,2 +-4.726,2 +-4.617,2 +-4.509,2 +-4.396,2 +-4.284,2 +-4.17,2 +-4.055,2 +-3.948,2 +-3.834,2 +-3.726,2 +-3.62,2 +-3.504,2 +-3.389,2 +-3.276,2 +-3.168,2 +-3.062,2 +-2.955,2 +-2.847,2 +-2.734,2 +-2.618,2 +-2.513,2 +-2.399,2 +-2.292,2 +-2.184,2 +-2.067,2 +-1.95,2 +-1.834,2 +-1.728,2 +-1.614,2 +-1.499,2 +-1.388,2 +-1.272,2 +-1.165,2 +-1.059,2 +-0.952,2 +-0.837,2 +-0.725,2 +-0.609,2 +-0.505,2 +-0.392,2 +-0.286,2 +-0.174,2 +-0.067,2 +0.041,2 +0.144,2 +0.252,2 +0.357,2 +0.473,2 +0.585,2 +0.698,2 +0.811,2 +0.924,2 +1.033,2 +1.137,2 +1.24,2 +1.354,2 +1.466,2 +1.581,2 +1.691,2 +1.805,2 +1.915,2 +2.029,2 +2.143,2 +2.253,2 +2.367,2 +2.482,2 +2.595,2 +2.708,2 +2.813,3 +2.917,3 +3.024,3 +3.135,3 +3.241,3 +3.354,3 +3.469,3 +3.573,3 +3.684,2 +3.792,2 +3.899,2 +4.005,2 +4.115,2 +4.226,2 +4.34,2 +4.446,2 +4.56,2 +4.674,2 +4.785,2 +4.898,2 +5.012,2 +5.126,2 +5.233,2 +5.336,2 +5.441,2 +5.554,2 +5.666,2 +5.774,2 +5.888,2 +6.001,2 +6.117,2 +6.229,2 +6.341,2 +6.455,2 +6.559,2 +6.665,3 +6.783,3 +6.888,3 +6.999,3 +7.107,3 +7.215,3 +7.319,3 +7.43,3 +7.541,3 +7.648,3 +7.753,2 +7.864,2 +7.973,2 +8.086,2 +8.199,2 +8.306,2 +8.412,2 +8.525,2 +8.635,2 +8.741,2 +8.853,2 +8.966,2 +9.082,2 +9.189,2 +9.3,2 +9.416,2 +9.522,2 +9.634,2 +9.749,2 +9.865,2 +9.975,2 +10.089,2 +10.201,2 +10.316,2 +10.426,2 +10.533,2 +10.638,2 +10.754,2 +10.865,2 +10.974,2 +11.083,2 +11.197,2 +11.308,2 +11.416,2 +11.531,2 +11.642,2 +11.752,2 +11.857,2 +11.973,2 +12.083,2 +12.195,2 +12.31,2 +12.418,2 +12.534,2 +12.647,2 +12.752,2 +12.865,2 +12.979,2 +13.093,2 +13.199,2 +13.306,2 +13.417,2 +13.523,2 +13.633,2 +13.742,2 +13.854,2 +13.96,2 +14.069,2 +14.175,2 +14.288,2 +14.402,2 +14.514,2 +14.627,2 +14.736,2 +14.842,2 +14.953,2 +15.065,2 +15.176,2 +15.289,2 +15.402,2 +15.515,2 +15.624,2 +15.733,2 +15.839,2 +15.955,2 +16.066,2 +16.173,2 +16.283,2 +16.391,2 +16.499,2 +16.611,2 +16.725,2 +16.833,2 +16.941,2 +17.053,2 +17.168,2 +17.283,2 +17.398,2 +17.508,2 +17.618,2 +17.731,2 +17.845,2 +17.958,2 +18.073,2 +18.188,2 +18.302,2 +18.417,2 +18.525,2 +18.634,2 +18.741,2 +18.858,2 +18.973,2 +19.085,2 +19.191,2 +19.296,2 +19.413,2 +19.527,2 +19.642,2 +19.749,2 +19.865,2 +19.979,2 +20.09,2 +20.198,2 +20.307,2 +20.418,2 +20.529,2 +20.641,2 +20.756,2 +20.863,2 +20.973,2 +21.089,2 +21.198,2 +21.31,2 +21.426,2 +21.541,2 +21.651,2 +21.767,2 +21.883,2 +21.987,2 +22.1,2 +22.215,2 +22.33,2 +22.437,2 +22.549,2 +22.666,2 +22.774,2 +22.882,2 +22.991,2 +23.099,2 +23.208,2 +23.316,2 +23.433,2 +23.547,2 +23.662,2 +23.774,2 +23.889,2 +24.002,2 +24.116,2 +24.229,2 +24.344,2 +24.456,2 +24.563,2 +24.675,2 +24.791,2 +24.905,2 +25.017,2 +25.129,2 +25.241,2 +25.353,2 +25.458,2 +25.57,2 +25.683,2 +25.795,2 +25.91,2 +26.016,2 +26.123,2 +26.228,2 +26.334,2 +26.446,2 +26.562,2 +26.679,2 +26.789,2 +26.901,2 +27.016,2 +27.129,2 +27.238,2 +27.352,2 +27.457,2 +27.563,2 +27.678,2 +27.787,2 +27.892,2 +27.999,2 +28.105,2 +28.219,2 +28.333,2 +28.441,2 +28.547,2 +28.656,2 +28.762,2 +28.877,2 +28.985,2 +29.098,2 +29.208,2 +29.313,2 +29.422,2 +29.535,2 +29.647,2 +29.762,2 +29.878,2 +29.983,2 +30.089,2 +30.204,2 +30.309,2 +30.42,2 +30.527,2 +30.638,2 +30.753,2 +30.861,2 +30.974,2 +31.082,2 +31.189,2 +31.3,2 +31.405,2 +31.52,2 +31.635,2 +31.749,2 +31.863,2 +31.978,2 +32.091,2 +32.203,2 +32.318,2 +32.425,2 +32.54,2 +32.654,2 +32.76,2 +32.866,2 +32.975,2 +33.09,2 +33.199,2 +33.304,2 +33.415,2 +33.529,2 +33.642,2 +33.754,2 +33.861,2 +33.966,2 +34.079,2 +34.193,2 +34.306,2 +34.412,2 +34.526,2 +34.633,2 +34.741,2 +34.849,2 +34.966,2 +35.081,2 +35.191,2 +35.304,2 +35.419,2 +35.528,2 +35.642,2 +35.757,2 +35.866,2 +35.981,2 +36.091,2 +36.197,2 +36.309,2 +36.425,2 +36.541,2 +36.653,2 +36.768,2 +36.875,2 +36.983,2 +37.092,2 +37.2,2 +37.312,2 +37.417,2 +37.525,2 +37.639,2 +37.746,2 +37.861,2 +37.967,2 +38.083,2 +38.194,2 +38.309,2 +38.417,2 +38.53,2 +38.641,2 +38.748,2 +38.856,2 +38.971,2 +39.08,2 +39.191,2 +39.307,2 +39.423,2 +39.537,2 +39.652,2 +39.758,2 +39.865,2 +39.979,2 +40.094,2 +40.209,2 +40.322,2 +40.431,2 +40.541,2 +40.65,2 +40.758,2 +40.868,2 +40.977,2 +41.084,2 +41.189,2 +41.299,2 +41.408,2 +41.516,2 +41.63,2 +41.746,2 +41.858,2 +41.965,2 +42.075,2 +42.188,2 +42.305,2 +42.417,2 +42.533,2 +42.647,2 +42.762,2 +42.875,2 +42.981,2 +43.095,2 +43.212,2 +43.326,2 +43.43,2 +43.537,2 +43.645,2 +43.757,2 +43.869,2 +43.977,2 +44.093,2 +44.2,2 +44.307,2 +44.415,2 +44.525,2 +44.635,2 +44.747,2 +44.859,2 +44.974,2 +45.088,2 +45.194,2 +45.307,2 +45.423,2 +45.533,2 +45.646,2 +45.752,2 +45.857,2 +45.964,2 +46.079,2 +46.191,2 +46.3,2 +46.414,2 +46.518,2 +46.629,2 +46.735,2 +46.841,2 +46.949,2 +47.058,2 +47.167,2 +47.275,2 +47.389,2 +47.496,2 +47.611,2 +47.725,2 +47.84,2 +47.948,2 +48.058,2 +48.173,2 +48.286,2 +48.393,2 +48.507,2 +48.618,2 +48.734,2 +48.84,2 +48.948,2 +49.063,2 +49.178,2 +49.291,2 +49.4,2 +49.507,2 +49.617,2 +49.729,2 +49.835,2 +49.945,2 +50.06,2 +50.175,2 +50.29,2 +50.405,2 +50.516,2 +50.632,2 +50.745,2 +50.859,2 +50.976,2 +51.082,2 +51.194,2 +51.298,2 +51.408,2 +51.513,2 +51.62,2 +51.73,2 +51.845,2 +51.954,2 +52.068,2 +52.176,2 +52.283,2 +52.39,2 +52.496,2 +52.607,2 +52.715,2 +52.826,2 +52.94,2 +53.053,2 +53.16,2 +53.274,2 +53.388,2 +53.499,2 +53.612,2 +53.725,2 +53.83,2 +53.941,2 +54.046,2 +54.151,2 +54.264,2 +54.375,2 +54.49,2 +54.604,2 +54.714,2 +54.825,2 +54.936,2 +55.049,2 +55.163,2 +55.272,2 +55.388,2 +55.499,2 +55.614,2 +55.731,2 +55.835,2 +55.941,2 +56.057,2 +56.168,2 +56.275,2 +56.391,2 +56.506,2 +56.62,2 +56.733,2 +56.844,2 +56.959,2 +57.073,2 +57.178,2 +57.291,2 +57.406,2 +57.51,2 +57.626,2 +57.73,2 +57.841,2 +57.956,2 +58.071,2 +58.18,2 +58.293,2 +58.4,2 +58.506,2 +58.623,2 +58.733,2 +58.841,2 +58.958,2 +59.065,2 +59.175,2 +59.29,2 +59.394,2 +59.507,2 +59.621,2 +59.733,2 +59.847,2 +59.958,2 +60.066,2 +60.181,2 +60.296,2 +60.411,2 +60.518,2 +60.627,2 +60.733,2 +60.836,2 +60.943,2 +61.056,2 +61.166,2 +61.275,2 +61.383,2 +61.492,2 +61.604,2 +61.712,2 +61.825,2 +61.928,2 +62.04,2 +62.153,2 +62.269,2 +62.375,2 +62.489,2 +62.602,2 +62.719,2 +62.828,2 +62.941,2 +63.057,2 +63.168,2 +63.275,2 +63.389,2 +63.497,2 +63.611,2 +63.717,2 +63.825,2 +63.94,2 +64.05,2 +64.165,2 +64.272,2 +64.386,2 +64.49,2 +64.596,2 +64.702,2 +64.813,2 +64.927,2 +65.043,2 +65.152,2 +65.264,2 +65.374,2 +65.484,2 +65.596,2 +65.711,2 +65.826,2 +65.935,2 +66.041,2 +66.152,2 +66.266,2 +66.374,2 +66.483,2 +66.598,2 +66.707,2 +66.816,2 +66.931,2 +67.036,2 +67.146,2 +67.259,2 +67.365,2 +67.471,2 +67.586,2 +67.691,2 +67.802,2 +67.915,2 +68.025,2 +68.138,2 +68.244,2 +68.357,2 +68.463,2 +68.571,2 +68.686,2 +68.802,2 +68.919,2 +69.031,2 +69.142,2 +69.248,2 +69.361,2 +69.471,2 +69.578,2 +69.691,2 +69.801,2 +69.908,2 +70.016,2 +70.124,2 +70.239,2 +70.353,2 +70.467,2 +70.579,2 +70.691,2 +70.8,2 +70.906,2 +71.021,2 +71.137,2 +71.247,2 +71.355,2 +71.471,2 +71.586,2 +71.693,2 +71.799,2 +71.906,2 +72.022,2 +72.133,2 +72.246,2 +72.359,2 +72.474,2 +72.585,2 +72.694,2 +72.808,2 +72.923,2 +73.033,2 +73.142,2 +73.253,2 +73.365,2 +73.48,2 +73.592,2 +73.703,2 +73.811,2 +73.927,2 +74.033,2 +74.146,2 +74.258,2 +74.366,2 +74.479,2 +74.587,2 +74.694,2 +74.802,2 +74.912,2 +75.023,2 +75.135,2 +75.241,2 +75.353,2 +75.465,2 diff --git a/evidence/rolling-from-8d28d45/probe.csv b/evidence/rolling-from-8d28d45/probe.csv new file mode 100644 index 00000000..f93270b6 --- /dev/null +++ b/evidence/rolling-from-8d28d45/probe.csv @@ -0,0 +1,898 @@ +t_rel_s,status,latency_ms,instance +-25.776,201,16.9,00548d9b229e +-25.655,201,19.3,7a4c320f9b13 +-25.535,201,27.3,00548d9b229e +-25.407,201,4.9,00548d9b229e +-25.292,201,5.4,00548d9b229e +-25.185,201,8.3,7a4c320f9b13 +-25.076,201,5.5,7a4c320f9b13 +-24.96,201,4.5,00548d9b229e +-24.848,201,4.6,00548d9b229e +-24.742,201,19.0,00548d9b229e +-24.622,201,6.2,7a4c320f9b13 +-24.508,201,9.3,7a4c320f9b13 +-24.389,201,6.9,00548d9b229e +-24.28,201,7.5,00548d9b229e +-24.163,201,8.6,7a4c320f9b13 +-24.049,201,7.0,7a4c320f9b13 +-23.934,201,7.2,7a4c320f9b13 +-23.827,201,7.7,7a4c320f9b13 +-23.717,201,5.1,7a4c320f9b13 +-23.609,201,7.5,7a4c320f9b13 +-23.494,201,9.1,00548d9b229e +-23.38,201,9.4,00548d9b229e +-23.267,201,4.2,00548d9b229e +-23.162,201,5.5,00548d9b229e +-23.051,201,6.5,00548d9b229e +-22.943,201,6.8,00548d9b229e +-22.834,201,8.4,00548d9b229e +-22.717,201,6.4,00548d9b229e +-22.601,201,7.6,7a4c320f9b13 +-22.492,201,8.7,7a4c320f9b13 +-22.376,201,6.3,7a4c320f9b13 +-22.267,201,7.2,7a4c320f9b13 +-22.156,201,6.2,7a4c320f9b13 +-22.042,201,6.4,00548d9b229e +-21.931,201,7.9,00548d9b229e +-21.813,201,6.0,7a4c320f9b13 +-21.701,201,5.9,7a4c320f9b13 +-21.594,201,6.2,00548d9b229e +-21.485,201,6.1,00548d9b229e +-21.377,201,7.3,00548d9b229e +-21.259,201,6.5,7a4c320f9b13 +-21.146,201,6.8,7a4c320f9b13 +-21.038,201,8.7,7a4c320f9b13 +-20.919,201,6.2,00548d9b229e +-20.805,201,7.1,00548d9b229e +-20.691,201,6.5,7a4c320f9b13 +-20.584,201,6.6,00548d9b229e +-20.474,201,7.2,00548d9b229e +-20.357,201,6.7,7a4c320f9b13 +-20.242,201,7.1,00548d9b229e +-20.134,201,6.0,7a4c320f9b13 +-20.021,201,6.8,00548d9b229e +-19.909,201,6.1,00548d9b229e +-19.793,201,5.4,00548d9b229e +-19.684,201,5.0,00548d9b229e +-19.57,201,4.9,7a4c320f9b13 +-19.456,201,6.2,7a4c320f9b13 +-19.341,201,6.2,00548d9b229e +-19.225,201,6.9,00548d9b229e +-19.11,201,7.7,7a4c320f9b13 +-18.992,201,8.0,00548d9b229e +-18.876,201,7.0,00548d9b229e +-18.767,201,6.8,00548d9b229e +-18.659,201,6.4,00548d9b229e +-18.543,201,7.4,7a4c320f9b13 +-18.425,201,8.2,7a4c320f9b13 +-18.31,201,7.3,00548d9b229e +-18.202,201,7.6,00548d9b229e +-18.091,201,4.1,00548d9b229e +-17.986,201,4.9,7a4c320f9b13 +-17.873,201,5.5,7a4c320f9b13 +-17.758,201,5.5,00548d9b229e +-17.645,201,5.4,00548d9b229e +-17.533,201,7.5,7a4c320f9b13 +-17.416,201,6.1,7a4c320f9b13 +-17.308,201,5.2,7a4c320f9b13 +-17.202,201,6.6,7a4c320f9b13 +-17.086,201,6.2,7a4c320f9b13 +-16.969,201,6.6,00548d9b229e +-16.854,201,6.4,7a4c320f9b13 +-16.742,201,6.4,7a4c320f9b13 +-16.627,201,6.9,7a4c320f9b13 +-16.516,201,7.7,7a4c320f9b13 +-16.404,201,6.2,00548d9b229e +-16.292,201,6.1,00548d9b229e +-16.18,201,7.0,00548d9b229e +-16.063,201,6.9,7a4c320f9b13 +-15.947,201,6.8,7a4c320f9b13 +-15.834,201,7.5,00548d9b229e +-15.716,201,5.4,00548d9b229e +-15.609,201,4.7,00548d9b229e +-15.503,201,6.2,00548d9b229e +-15.39,201,6.6,00548d9b229e +-15.274,201,6.5,7a4c320f9b13 +-15.166,201,7.1,00548d9b229e +-15.051,201,7.1,7a4c320f9b13 +-14.935,201,6.1,00548d9b229e +-14.825,201,6.6,00548d9b229e +-14.717,201,6.9,00548d9b229e +-14.601,201,7.8,7a4c320f9b13 +-14.486,201,6.6,7a4c320f9b13 +-14.373,201,7.1,7a4c320f9b13 +-14.259,201,7.3,00548d9b229e +-14.142,201,7.6,7a4c320f9b13 +-14.032,201,7.5,7a4c320f9b13 +-13.916,201,7.0,7a4c320f9b13 +-13.799,201,7.0,00548d9b229e +-13.684,201,7.5,00548d9b229e +-13.568,201,7.4,7a4c320f9b13 +-13.451,201,6.3,00548d9b229e +-13.341,201,7.7,00548d9b229e +-13.226,201,8.3,00548d9b229e +-13.114,201,7.2,00548d9b229e +-13.002,201,8.1,7a4c320f9b13 +-12.886,201,4.6,7a4c320f9b13 +-12.776,201,4.9,7a4c320f9b13 +-12.668,201,5.3,7a4c320f9b13 +-12.552,201,7.0,00548d9b229e +-12.435,201,8.1,00548d9b229e +-12.317,201,8.5,7a4c320f9b13 +-12.205,201,7.4,7a4c320f9b13 +-12.098,201,7.2,7a4c320f9b13 +-11.986,201,8.1,7a4c320f9b13 +-11.87,201,8.5,00548d9b229e +-11.752,201,7.1,00548d9b229e +-11.644,201,5.4,00548d9b229e +-11.535,201,6.6,00548d9b229e +-11.423,201,5.7,00548d9b229e +-11.311,201,6.2,7a4c320f9b13 +-11.198,201,6.8,7a4c320f9b13 +-11.084,201,6.8,7a4c320f9b13 +-10.967,201,6.2,00548d9b229e +-10.855,201,6.8,00548d9b229e +-10.742,201,6.3,00548d9b229e +-10.631,201,6.0,7a4c320f9b13 +-10.523,201,5.5,00548d9b229e +-10.409,201,6.5,7a4c320f9b13 +-10.299,201,7.2,7a4c320f9b13 +-10.185,201,8.9,7a4c320f9b13 +-10.07,201,6.9,00548d9b229e +-9.953,201,6.4,00548d9b229e +-9.837,201,7.5,7a4c320f9b13 +-9.72,201,7.7,00548d9b229e +-9.606,201,4.1,00548d9b229e +-9.501,201,5.7,00548d9b229e +-9.387,201,6.8,00548d9b229e +-9.271,201,6.9,7a4c320f9b13 +-9.156,201,7.4,7a4c320f9b13 +-9.043,201,7.8,7a4c320f9b13 +-8.925,201,7.7,00548d9b229e +-8.809,201,6.8,7a4c320f9b13 +-8.694,201,7.5,7a4c320f9b13 +-8.576,201,6.6,00548d9b229e +-8.46,201,7.9,7a4c320f9b13 +-8.347,201,7.3,7a4c320f9b13 +-8.234,201,7.7,7a4c320f9b13 +-8.126,201,8.0,7a4c320f9b13 +-8.015,201,6.7,7a4c320f9b13 +-7.901,201,6.9,00548d9b229e +-7.784,201,6.2,00548d9b229e +-7.677,201,4.3,00548d9b229e +-7.563,201,5.9,00548d9b229e +-7.451,201,6.0,7a4c320f9b13 +-7.343,201,5.8,00548d9b229e +-7.236,201,5.7,00548d9b229e +-7.125,201,5.6,00548d9b229e +-7.013,201,7.5,00548d9b229e +-6.901,201,7.8,7a4c320f9b13 +-6.792,201,7.4,00548d9b229e +-6.675,201,6.8,7a4c320f9b13 +-6.568,201,8.1,7a4c320f9b13 +-6.459,201,8.0,7a4c320f9b13 +-6.347,201,7.8,7a4c320f9b13 +-6.233,201,5.5,7a4c320f9b13 +-6.118,201,8.3,00548d9b229e +-6.005,201,8.4,00548d9b229e +-5.896,201,5.9,00548d9b229e +-5.78,201,10.3,7a4c320f9b13 +-5.667,201,7.1,7a4c320f9b13 +-5.559,201,7.0,7a4c320f9b13 +-5.442,201,7.3,7a4c320f9b13 +-5.326,201,5.7,00548d9b229e +-5.217,201,7.3,00548d9b229e +-5.101,201,7.5,7a4c320f9b13 +-4.992,201,7.5,00548d9b229e +-4.879,201,7.2,7a4c320f9b13 +-4.763,201,6.5,7a4c320f9b13 +-4.651,201,6.7,00548d9b229e +-4.534,201,8.2,00548d9b229e +-4.416,201,6.7,7a4c320f9b13 +-4.306,201,7.1,7a4c320f9b13 +-4.198,201,6.8,7a4c320f9b13 +-4.084,201,8.1,7a4c320f9b13 +-3.966,201,8.1,00548d9b229e +-3.849,201,8.4,7a4c320f9b13 +-3.731,201,7.5,00548d9b229e +-3.622,201,10.5,7a4c320f9b13 +-3.505,201,7.4,00548d9b229e +-3.39,201,7.9,7a4c320f9b13 +-3.279,201,7.4,7a4c320f9b13 +-3.168,201,8.2,7a4c320f9b13 +-3.059,201,6.2,7a4c320f9b13 +-2.947,201,6.4,7a4c320f9b13 +-2.838,201,6.3,7a4c320f9b13 +-2.731,201,6.8,7a4c320f9b13 +-2.615,201,4.4,7a4c320f9b13 +-2.501,201,3.9,00548d9b229e +-2.39,201,5.0,00548d9b229e +-2.276,201,5.3,00548d9b229e +-2.167,201,6.9,00548d9b229e +-2.057,201,6.2,00548d9b229e +-1.942,201,6.4,00548d9b229e +-1.835,201,7.4,00548d9b229e +-1.725,201,6.4,00548d9b229e +-1.608,201,5.7,7a4c320f9b13 +-1.497,201,4.0,7a4c320f9b13 +-1.385,201,5.3,7a4c320f9b13 +-1.273,201,7.8,00548d9b229e +-1.163,201,5.5,7a4c320f9b13 +-1.058,201,6.6,7a4c320f9b13 +-0.941,201,6.6,00548d9b229e +-0.834,201,5.7,7a4c320f9b13 +-0.725,201,7.8,7a4c320f9b13 +-0.612,201,9.0,00548d9b229e +-0.501,201,7.1,00548d9b229e +-0.384,201,6.4,00548d9b229e +-0.272,201,6.8,7a4c320f9b13 +-0.163,201,5.9,00548d9b229e +-0.057,201,6.3,00548d9b229e +0.058,201,4.5,00548d9b229e +0.167,201,4.0,00548d9b229e +0.275,201,5.0,00548d9b229e +0.38,201,5.2,00548d9b229e +0.493,201,4.1,00548d9b229e +0.602,201,4.6,00548d9b229e +0.716,201,4.3,00548d9b229e +0.83,201,4.9,7a4c320f9b13 +0.942,201,5.4,7a4c320f9b13 +1.05,201,3.6,7a4c320f9b13 +1.155,201,4.8,00548d9b229e +1.265,201,3.9,00548d9b229e +1.379,201,4.3,00548d9b229e +1.485,201,5.5,00548d9b229e +1.599,201,4.3,00548d9b229e +1.706,201,3.5,00548d9b229e +1.819,201,4.5,7a4c320f9b13 +1.934,201,7.6,7a4c320f9b13 +2.051,201,6.1,00548d9b229e +2.161,201,7.0,00548d9b229e +2.279,201,4.9,00548d9b229e +2.391,201,4.5,7a4c320f9b13 +2.496,201,4.4,00548d9b229e +2.603,201,4.4,00548d9b229e +2.715,201,5.3,00548d9b229e +2.827,201,4.5,28af1eaa77c8 +2.933,201,4.4,28af1eaa77c8 +3.042,201,5.1,28af1eaa77c8 +3.147,201,4.8,00548d9b229e +3.255,201,5.2,00548d9b229e +3.369,201,507.5,28af1eaa77c8 +3.981,201,4.5,00548d9b229e +4.091,201,4.9,28af1eaa77c8 +4.199,201,4.1,00548d9b229e +4.308,201,6.6,00548d9b229e +4.418,201,4.9,28af1eaa77c8 +4.523,201,6.9,28af1eaa77c8 +4.639,201,7.0,00548d9b229e +4.755,201,5.2,00548d9b229e +4.861,201,4.8,00548d9b229e +4.975,201,4.8,00548d9b229e +5.085,201,7.0,00548d9b229e +5.193,201,6.2,00548d9b229e +5.3,201,5.8,00548d9b229e +5.406,201,5.1,28af1eaa77c8 +5.512,201,6.2,28af1eaa77c8 +5.627,201,7.7,28af1eaa77c8 +5.744,201,6.6,00548d9b229e +5.859,201,7.8,00548d9b229e +5.973,201,6.7,28af1eaa77c8 +6.089,201,7.2,28af1eaa77c8 +6.199,201,8.4,28af1eaa77c8 +6.315,201,6.5,00548d9b229e +6.431,201,6.4,28af1eaa77c8 +6.547,201,7.4,28af1eaa77c8 +6.664,201,10.7,00548d9b229e +6.783,201,4.6,00548d9b229e +6.889,201,6.1,00548d9b229e +7.002,201,4.5,00548d9b229e +7.107,201,6.2,00548d9b229e +7.215,201,5.5,f3f2f7801b56 +7.323,201,6.0,f3f2f7801b56 +7.432,201,5.3,f3f2f7801b56 +7.546,201,3.7,28af1eaa77c8 +7.657,201,3.9,28af1eaa77c8 +7.763,201,3.2,f3f2f7801b56 +7.866,201,4.2,28af1eaa77c8 +7.974,201,4.8,f3f2f7801b56 +8.086,201,7.7,f3f2f7801b56 +8.197,201,7.3,28af1eaa77c8 +8.306,201,8.2,28af1eaa77c8 +8.423,201,5.1,f3f2f7801b56 +8.533,201,6.2,f3f2f7801b56 +8.642,201,6.9,f3f2f7801b56 +8.752,201,5.0,f3f2f7801b56 +8.86,201,5.6,f3f2f7801b56 +8.966,201,8.7,f3f2f7801b56 +9.081,201,7.4,f3f2f7801b56 +9.191,201,6.1,f3f2f7801b56 +9.298,201,8.1,28af1eaa77c8 +9.415,201,6.8,f3f2f7801b56 +9.524,201,6.9,f3f2f7801b56 +9.64,201,5.9,28af1eaa77c8 +9.747,201,8.4,f3f2f7801b56 +9.865,201,11.0,28af1eaa77c8 +9.978,201,4.5,28af1eaa77c8 +10.089,201,5.8,f3f2f7801b56 +10.201,201,6.4,f3f2f7801b56 +10.317,201,6.5,f3f2f7801b56 +10.424,201,7.5,f3f2f7801b56 +10.541,201,5.6,28af1eaa77c8 +10.653,201,6.8,28af1eaa77c8 +10.769,201,5.2,f3f2f7801b56 +10.878,201,5.1,28af1eaa77c8 +10.994,201,3.9,f3f2f7801b56 +11.102,201,6.2,f3f2f7801b56 +11.216,201,7.0,f3f2f7801b56 +11.325,201,6.5,f3f2f7801b56 +11.433,201,6.5,f3f2f7801b56 +11.54,201,7.7,f3f2f7801b56 +11.654,201,6.4,f3f2f7801b56 +11.765,201,5.5,f3f2f7801b56 +11.875,201,7.2,f3f2f7801b56 +11.991,201,6.3,28af1eaa77c8 +12.107,201,7.1,28af1eaa77c8 +12.218,201,7.5,28af1eaa77c8 +12.336,201,6.7,f3f2f7801b56 +12.446,201,6.4,f3f2f7801b56 +12.562,201,6.2,28af1eaa77c8 +12.678,201,4.3,28af1eaa77c8 +12.793,201,6.6,f3f2f7801b56 +12.908,201,5.1,f3f2f7801b56 +13.023,201,4.5,28af1eaa77c8 +13.135,201,7.4,28af1eaa77c8 +13.249,201,8.9,f3f2f7801b56 +13.36,201,7.2,f3f2f7801b56 +13.473,201,6.4,f3f2f7801b56 +13.586,201,7.4,f3f2f7801b56 +13.699,201,7.5,28af1eaa77c8 +13.816,201,7.6,28af1eaa77c8 +13.928,201,7.7,f3f2f7801b56 +14.045,201,6.6,f3f2f7801b56 +14.157,201,8.9,f3f2f7801b56 +14.274,201,6.9,28af1eaa77c8 +14.383,201,6.8,28af1eaa77c8 +14.491,201,8.3,28af1eaa77c8 +14.602,201,6.5,28af1eaa77c8 +14.713,201,7.6,28af1eaa77c8 +14.827,201,7.8,f3f2f7801b56 +14.945,201,6.3,f3f2f7801b56 +15.058,201,6.8,28af1eaa77c8 +15.165,201,4.3,f3f2f7801b56 +15.27,201,4.6,f3f2f7801b56 +15.376,201,5.1,28af1eaa77c8 +15.486,201,6.4,28af1eaa77c8 +15.594,201,7.4,28af1eaa77c8 +15.705,201,6.7,28af1eaa77c8 +15.822,201,8.3,f3f2f7801b56 +15.932,201,7.1,f3f2f7801b56 +16.04,201,7.9,f3f2f7801b56 +16.158,201,7.6,f3f2f7801b56 +16.268,201,7.9,28af1eaa77c8 +16.386,201,8.1,28af1eaa77c8 +16.498,201,7.7,28af1eaa77c8 +16.613,201,6.1,f3f2f7801b56 +16.724,201,7.9,f3f2f7801b56 +16.841,201,3.8,28af1eaa77c8 +16.949,201,5.0,f3f2f7801b56 +17.058,201,5.2,f3f2f7801b56 +17.171,201,7.7,28af1eaa77c8 +17.28,201,8.5,28af1eaa77c8 +17.399,201,7.4,f3f2f7801b56 +17.507,201,8.4,f3f2f7801b56 +17.618,201,8.9,f3f2f7801b56 +17.737,201,6.0,f3f2f7801b56 +17.846,201,5.7,f3f2f7801b56 +17.961,201,5.3,28af1eaa77c8 +18.073,201,5.1,28af1eaa77c8 +18.187,201,6.1,f3f2f7801b56 +18.298,201,6.6,f3f2f7801b56 +18.406,201,6.1,28af1eaa77c8 +18.519,201,8.9,f3f2f7801b56 +18.633,201,8.4,f3f2f7801b56 +18.749,201,6.9,28af1eaa77c8 +18.866,201,5.2,28af1eaa77c8 +18.978,201,5.7,f3f2f7801b56 +19.086,201,6.1,28af1eaa77c8 +19.196,201,5.6,28af1eaa77c8 +19.309,201,6.8,f3f2f7801b56 +19.417,201,7.0,28af1eaa77c8 +19.532,201,5.8,f3f2f7801b56 +19.648,201,6.5,f3f2f7801b56 +19.758,201,7.2,28af1eaa77c8 +19.872,201,7.9,28af1eaa77c8 +19.981,201,7.4,28af1eaa77c8 +20.092,201,7.4,28af1eaa77c8 +20.208,201,4.0,28af1eaa77c8 +20.318,201,3.9,28af1eaa77c8 +20.431,201,5.2,f3f2f7801b56 +20.545,201,6.4,f3f2f7801b56 +20.656,201,6.9,f3f2f7801b56 +20.766,201,5.0,f3f2f7801b56 +20.878,201,6.9,f3f2f7801b56 +20.995,201,5.8,28af1eaa77c8 +21.103,201,5.9,28af1eaa77c8 +21.218,201,6.7,f3f2f7801b56 +21.332,201,7.0,f3f2f7801b56 +21.44,201,8.1,f3f2f7801b56 +21.55,201,6.4,f3f2f7801b56 +21.657,201,6.7,28af1eaa77c8 +21.767,201,8.9,f3f2f7801b56 +21.886,201,4.3,f3f2f7801b56 +21.991,201,4.2,f3f2f7801b56 +22.099,201,7.2,f3f2f7801b56 +22.216,201,7.6,f3f2f7801b56 +22.333,201,6.0,28af1eaa77c8 +22.439,201,6.7,f3f2f7801b56 +22.555,201,5.4,28af1eaa77c8 +22.666,201,9.0,28af1eaa77c8 +22.782,201,6.9,f3f2f7801b56 +22.891,201,5.1,f3f2f7801b56 +22.999,201,4.2,28af1eaa77c8 +23.113,201,6.6,f3f2f7801b56 +23.23,201,5.4,28af1eaa77c8 +23.341,201,5.9,28af1eaa77c8 +23.453,201,6.5,28af1eaa77c8 +23.568,201,7.2,28af1eaa77c8 +23.683,201,5.6,f3f2f7801b56 +23.798,201,6.5,f3f2f7801b56 +23.907,201,7.7,f3f2f7801b56 +24.016,201,6.6,f3f2f7801b56 +24.123,201,5.6,f3f2f7801b56 +24.235,201,5.8,f3f2f7801b56 +24.349,201,7.4,28af1eaa77c8 +24.466,201,5.3,28af1eaa77c8 +24.573,201,6.3,28af1eaa77c8 +24.683,201,6.4,28af1eaa77c8 +24.791,201,18.8,28af1eaa77c8 +24.916,201,4.4,f3f2f7801b56 +25.028,201,5.1,f3f2f7801b56 +25.143,201,4.9,28af1eaa77c8 +25.25,201,5.9,28af1eaa77c8 +25.361,201,5.2,28af1eaa77c8 +25.475,201,3.9,28af1eaa77c8 +25.584,201,4.6,28af1eaa77c8 +25.699,201,6.5,f3f2f7801b56 +25.806,201,6.1,28af1eaa77c8 +25.92,201,6.8,f3f2f7801b56 +26.032,201,7.9,f3f2f7801b56 +26.149,201,7.3,28af1eaa77c8 +26.266,201,6.7,28af1eaa77c8 +26.374,201,7.3,28af1eaa77c8 +26.482,201,8.4,28af1eaa77c8 +26.599,201,8.2,f3f2f7801b56 +26.709,201,7.1,f3f2f7801b56 +26.826,201,7.0,f3f2f7801b56 +26.942,201,8.3,28af1eaa77c8 +27.058,201,5.8,28af1eaa77c8 +27.166,201,5.9,28af1eaa77c8 +27.282,201,6.1,f3f2f7801b56 +27.39,201,4.8,f3f2f7801b56 +27.502,201,6.7,f3f2f7801b56 +27.618,201,5.9,28af1eaa77c8 +27.733,201,4.2,28af1eaa77c8 +27.841,201,5.2,28af1eaa77c8 +27.952,201,5.0,28af1eaa77c8 +28.058,201,5.2,f3f2f7801b56 +28.164,201,4.6,f3f2f7801b56 +28.278,201,5.2,f3f2f7801b56 +28.391,201,7.4,28af1eaa77c8 +28.501,201,6.5,28af1eaa77c8 +28.617,201,7.1,28af1eaa77c8 +28.726,201,7.7,28af1eaa77c8 +28.843,201,7.0,f3f2f7801b56 +28.952,201,8.1,f3f2f7801b56 +29.062,201,5.1,f3f2f7801b56 +29.172,201,5.8,f3f2f7801b56 +29.282,201,7.6,f3f2f7801b56 +29.391,201,7.0,f3f2f7801b56 +29.499,201,9.7,f3f2f7801b56 +29.613,201,6.9,f3f2f7801b56 +29.728,201,7.0,28af1eaa77c8 +29.837,201,7.4,f3f2f7801b56 +29.949,201,7.8,28af1eaa77c8 +30.066,201,6.6,28af1eaa77c8 +30.181,201,7.2,f3f2f7801b56 +30.297,201,7.7,f3f2f7801b56 +30.415,201,6.9,28af1eaa77c8 +30.523,201,6.5,28af1eaa77c8 +30.634,201,5.5,28af1eaa77c8 +30.741,201,5.1,28af1eaa77c8 +30.853,201,5.1,28af1eaa77c8 +30.969,201,6.6,f3f2f7801b56 +31.083,201,6.4,f3f2f7801b56 +31.191,201,4.1,f3f2f7801b56 +31.299,201,6.9,f3f2f7801b56 +31.41,201,6.3,f3f2f7801b56 +31.524,201,6.7,f3f2f7801b56 +31.633,201,8.7,f3f2f7801b56 +31.75,201,8.4,28af1eaa77c8 +31.859,201,10.2,28af1eaa77c8 +31.978,201,8.4,f3f2f7801b56 +32.088,201,8.4,f3f2f7801b56 +32.203,201,9.1,f3f2f7801b56 +32.316,201,7.9,f3f2f7801b56 +32.431,201,6.4,28af1eaa77c8 +32.54,201,9.2,28af1eaa77c8 +32.659,201,5.7,f3f2f7801b56 +32.775,201,6.6,f3f2f7801b56 +32.883,201,6.9,f3f2f7801b56 +32.996,201,6.0,f3f2f7801b56 +33.108,201,7.4,f3f2f7801b56 +33.216,201,4.2,f3f2f7801b56 +33.321,201,4.0,f3f2f7801b56 +33.435,201,6.1,f3f2f7801b56 +33.548,201,6.4,28af1eaa77c8 +33.659,201,6.2,28af1eaa77c8 +33.768,201,7.0,28af1eaa77c8 +33.885,201,5.3,28af1eaa77c8 +33.992,201,5.2,28af1eaa77c8 +34.107,201,4.3,28af1eaa77c8 +34.212,201,4.6,28af1eaa77c8 +34.323,201,5.1,28af1eaa77c8 +34.431,201,6.4,28af1eaa77c8 +34.543,201,6.8,28af1eaa77c8 +34.652,201,7.4,28af1eaa77c8 +34.766,201,6.2,28af1eaa77c8 +34.881,201,6.6,f3f2f7801b56 +34.998,201,8.3,f3f2f7801b56 +35.116,201,7.6,28af1eaa77c8 +35.225,201,5.6,28af1eaa77c8 +35.332,201,4.9,28af1eaa77c8 +35.437,201,7.1,f3f2f7801b56 +35.553,201,6.8,28af1eaa77c8 +35.665,201,6.4,28af1eaa77c8 +35.775,201,4.2,28af1eaa77c8 +35.888,201,4.9,f3f2f7801b56 +35.998,201,5.3,28af1eaa77c8 +36.107,201,6.2,28af1eaa77c8 +36.216,201,6.1,28af1eaa77c8 +36.329,201,7.3,f3f2f7801b56 +36.445,201,6.5,f3f2f7801b56 +36.561,201,7.7,28af1eaa77c8 +36.676,201,6.9,28af1eaa77c8 +36.785,201,7.2,28af1eaa77c8 +36.893,201,6.2,28af1eaa77c8 +37.0,201,6.6,28af1eaa77c8 +37.116,201,7.0,28af1eaa77c8 +37.233,201,5.2,f3f2f7801b56 +37.348,201,5.6,f3f2f7801b56 +37.456,201,6.1,f3f2f7801b56 +37.567,201,7.1,f3f2f7801b56 +37.674,201,6.7,f3f2f7801b56 +37.784,201,6.4,f3f2f7801b56 +37.891,201,7.3,f3f2f7801b56 +38.008,201,8.4,28af1eaa77c8 +38.122,201,7.2,28af1eaa77c8 +38.233,201,7.1,28af1eaa77c8 +38.35,201,5.2,f3f2f7801b56 +38.465,201,5.2,f3f2f7801b56 +38.58,201,5.1,28af1eaa77c8 +38.696,201,5.9,28af1eaa77c8 +38.812,201,6.7,f3f2f7801b56 +38.924,201,6.9,f3f2f7801b56 +39.041,201,8.4,28af1eaa77c8 +39.158,201,7.5,f3f2f7801b56 +39.273,201,4.7,f3f2f7801b56 +39.379,201,6.9,f3f2f7801b56 +39.496,201,6.6,f3f2f7801b56 +39.605,201,8.5,f3f2f7801b56 +39.716,201,7.1,f3f2f7801b56 +39.831,201,8.1,28af1eaa77c8 +39.944,201,6.6,28af1eaa77c8 +40.057,201,5.8,28af1eaa77c8 +40.171,201,6.8,f3f2f7801b56 +40.286,201,8.7,f3f2f7801b56 +40.401,201,7.4,28af1eaa77c8 +40.517,201,6.9,28af1eaa77c8 +40.632,201,7.9,f3f2f7801b56 +40.75,201,9.5,28af1eaa77c8 +40.861,201,5.2,28af1eaa77c8 +40.971,201,4.2,28af1eaa77c8 +41.083,201,5.0,28af1eaa77c8 +41.193,201,5.8,28af1eaa77c8 +41.308,201,5.6,28af1eaa77c8 +41.417,201,4.7,28af1eaa77c8 +41.532,201,5.8,f3f2f7801b56 +41.643,201,5.6,f3f2f7801b56 +41.75,201,6.7,f3f2f7801b56 +41.858,201,7.2,f3f2f7801b56 +41.966,201,8.2,28af1eaa77c8 +42.082,201,6.0,f3f2f7801b56 +42.198,201,6.4,f3f2f7801b56 +42.308,201,5.2,f3f2f7801b56 +42.416,201,9.2,28af1eaa77c8 +42.534,201,6.3,28af1eaa77c8 +42.646,201,8.7,28af1eaa77c8 +42.754,201,6.8,28af1eaa77c8 +42.87,201,8.8,f3f2f7801b56 +42.983,201,6.3,f3f2f7801b56 +43.097,201,7.5,f3f2f7801b56 +43.215,201,5.7,28af1eaa77c8 +43.324,201,7.0,28af1eaa77c8 +43.439,201,5.1,f3f2f7801b56 +43.552,201,4.0,f3f2f7801b56 +43.662,201,4.1,f3f2f7801b56 +43.773,201,4.1,f3f2f7801b56 +43.885,201,4.9,f3f2f7801b56 +44.0,201,7.0,28af1eaa77c8 +44.115,201,6.4,28af1eaa77c8 +44.23,201,7.3,f3f2f7801b56 +44.347,201,6.9,f3f2f7801b56 +44.458,201,7.9,28af1eaa77c8 +44.567,201,7.4,f3f2f7801b56 +44.681,201,7.8,28af1eaa77c8 +44.797,201,7.2,28af1eaa77c8 +44.911,201,7.0,f3f2f7801b56 +45.019,201,7.6,f3f2f7801b56 +45.135,201,8.3,f3f2f7801b56 +45.248,201,7.9,28af1eaa77c8 +45.361,201,8.5,28af1eaa77c8 +45.478,201,6.8,f3f2f7801b56 +45.586,201,6.0,28af1eaa77c8 +45.702,201,6.9,f3f2f7801b56 +45.817,201,7.1,f3f2f7801b56 +45.925,201,8.7,f3f2f7801b56 +46.041,201,4.8,28af1eaa77c8 +46.154,201,5.0,28af1eaa77c8 +46.263,201,4.7,28af1eaa77c8 +46.375,201,5.0,28af1eaa77c8 +46.483,201,7.6,28af1eaa77c8 +46.6,201,7.5,f3f2f7801b56 +46.717,201,7.1,f3f2f7801b56 +46.829,201,7.6,28af1eaa77c8 +46.937,201,7.8,f3f2f7801b56 +47.053,201,8.4,28af1eaa77c8 +47.165,201,7.6,28af1eaa77c8 +47.282,201,5.8,f3f2f7801b56 +47.393,201,5.4,f3f2f7801b56 +47.499,201,6.8,f3f2f7801b56 +47.608,201,8.3,f3f2f7801b56 +47.726,201,7.1,f3f2f7801b56 +47.84,201,9.4,28af1eaa77c8 +47.957,201,7.4,28af1eaa77c8 +48.074,201,5.7,f3f2f7801b56 +48.183,201,7.6,f3f2f7801b56 +48.297,201,6.3,f3f2f7801b56 +48.414,201,6.6,28af1eaa77c8 +48.524,201,8.2,28af1eaa77c8 +48.639,201,7.2,f3f2f7801b56 +48.754,201,4.2,f3f2f7801b56 +48.866,201,5.4,f3f2f7801b56 +48.982,201,4.8,28af1eaa77c8 +49.096,201,6.6,28af1eaa77c8 +49.213,201,7.7,f3f2f7801b56 +49.331,201,9.5,28af1eaa77c8 +49.446,201,6.9,28af1eaa77c8 +49.558,201,4.9,28af1eaa77c8 +49.67,201,5.7,f3f2f7801b56 +49.786,201,6.0,f3f2f7801b56 +49.9,201,7.6,28af1eaa77c8 +50.016,201,6.8,28af1eaa77c8 +50.125,201,7.0,28af1eaa77c8 +50.233,201,6.8,28af1eaa77c8 +50.346,201,6.8,28af1eaa77c8 +50.463,201,7.5,f3f2f7801b56 +50.575,201,7.4,f3f2f7801b56 +50.685,201,8.5,f3f2f7801b56 +50.803,201,7.0,28af1eaa77c8 +50.916,201,6.3,28af1eaa77c8 +51.026,201,6.8,28af1eaa77c8 +51.142,201,7.1,f3f2f7801b56 +51.252,201,4.6,f3f2f7801b56 +51.366,201,4.9,f3f2f7801b56 +51.475,201,5.9,f3f2f7801b56 +51.59,201,5.8,28af1eaa77c8 +51.699,201,5.9,28af1eaa77c8 +51.815,201,5.7,28af1eaa77c8 +51.925,201,5.9,28af1eaa77c8 +52.033,201,7.7,28af1eaa77c8 +52.15,201,7.4,f3f2f7801b56 +52.258,201,8.2,f3f2f7801b56 +52.372,201,7.6,f3f2f7801b56 +52.482,201,8.4,f3f2f7801b56 +52.596,201,7.9,f3f2f7801b56 +52.708,201,9.0,28af1eaa77c8 +52.828,201,6.3,28af1eaa77c8 +52.941,201,7.2,f3f2f7801b56 +53.058,201,8.0,28af1eaa77c8 +53.173,201,5.3,28af1eaa77c8 +53.278,201,5.6,f3f2f7801b56 +53.387,201,7.0,f3f2f7801b56 +53.5,201,6.1,28af1eaa77c8 +53.612,201,6.4,28af1eaa77c8 +53.721,201,6.2,28af1eaa77c8 +53.833,201,4.5,28af1eaa77c8 +53.941,201,5.1,28af1eaa77c8 +54.05,201,5.7,28af1eaa77c8 +54.156,201,3.9,f3f2f7801b56 +54.268,201,5.7,f3f2f7801b56 +54.378,201,6.1,f3f2f7801b56 +54.491,201,8.2,28af1eaa77c8 +54.604,201,8.1,28af1eaa77c8 +54.713,201,8.9,28af1eaa77c8 +54.827,201,6.5,28af1eaa77c8 +54.935,201,7.9,28af1eaa77c8 +55.049,201,8.2,f3f2f7801b56 +55.167,201,7.0,f3f2f7801b56 +55.279,201,6.5,28af1eaa77c8 +55.388,201,10.8,28af1eaa77c8 +55.507,201,5.0,28af1eaa77c8 +55.614,201,7.4,28af1eaa77c8 +55.728,201,6.8,28af1eaa77c8 +55.844,201,5.5,f3f2f7801b56 +55.953,201,6.2,f3f2f7801b56 +56.066,201,6.3,f3f2f7801b56 +56.173,201,5.8,f3f2f7801b56 +56.278,201,6.3,28af1eaa77c8 +56.393,201,3.8,f3f2f7801b56 +56.505,201,6.1,f3f2f7801b56 +56.62,201,8.2,28af1eaa77c8 +56.733,201,10.1,28af1eaa77c8 +56.845,201,9.3,28af1eaa77c8 +56.955,201,8.8,28af1eaa77c8 +57.072,201,6.9,f3f2f7801b56 +57.186,201,6.4,f3f2f7801b56 +57.296,201,6.2,f3f2f7801b56 +57.404,201,7.7,f3f2f7801b56 +57.522,201,7.5,28af1eaa77c8 +57.638,201,5.4,28af1eaa77c8 +57.753,201,5.1,f3f2f7801b56 +57.863,201,6.5,f3f2f7801b56 +57.978,201,6.3,f3f2f7801b56 +58.092,201,7.1,28af1eaa77c8 +58.209,201,7.6,f3f2f7801b56 +58.327,201,6.7,f3f2f7801b56 +58.436,201,8.1,f3f2f7801b56 +58.553,201,7.1,28af1eaa77c8 +58.666,201,7.3,28af1eaa77c8 +58.783,201,6.4,f3f2f7801b56 +58.899,201,8.3,28af1eaa77c8 +59.015,201,5.0,28af1eaa77c8 +59.12,201,5.4,28af1eaa77c8 +59.227,201,4.6,f3f2f7801b56 +59.334,201,6.8,f3f2f7801b56 +59.451,201,6.9,28af1eaa77c8 +59.567,201,6.9,28af1eaa77c8 +59.68,201,5.3,f3f2f7801b56 +59.792,201,5.8,f3f2f7801b56 +59.907,201,6.3,f3f2f7801b56 +60.014,201,6.6,f3f2f7801b56 +60.128,201,7.3,28af1eaa77c8 +60.239,201,6.8,28af1eaa77c8 +60.353,201,8.0,28af1eaa77c8 +60.463,201,7.7,28af1eaa77c8 +60.576,201,5.5,28af1eaa77c8 +60.683,201,5.8,28af1eaa77c8 +60.795,201,6.9,28af1eaa77c8 +60.908,201,6.7,28af1eaa77c8 +61.023,201,5.9,f3f2f7801b56 +61.136,201,6.4,f3f2f7801b56 +61.251,201,8.0,28af1eaa77c8 +61.369,201,7.7,28af1eaa77c8 +61.479,201,8.2,f3f2f7801b56 +61.593,201,4.4,f3f2f7801b56 +61.707,201,6.7,f3f2f7801b56 +61.822,201,6.3,28af1eaa77c8 +61.933,201,5.1,28af1eaa77c8 +62.04,201,8.3,28af1eaa77c8 +62.151,201,8.5,28af1eaa77c8 +62.262,201,8.1,28af1eaa77c8 +62.371,201,7.5,28af1eaa77c8 +62.48,201,7.9,28af1eaa77c8 +62.596,201,7.4,28af1eaa77c8 +62.714,201,7.9,f3f2f7801b56 +62.827,201,6.5,f3f2f7801b56 +62.943,201,6.8,28af1eaa77c8 +63.057,201,8.0,28af1eaa77c8 +63.175,201,6.4,f3f2f7801b56 +63.291,201,6.8,28af1eaa77c8 +63.4,201,6.9,28af1eaa77c8 +63.51,201,6.2,28af1eaa77c8 +63.624,201,7.0,28af1eaa77c8 +63.739,201,5.0,28af1eaa77c8 +63.854,201,5.9,f3f2f7801b56 +63.963,201,8.0,f3f2f7801b56 +64.073,201,6.6,f3f2f7801b56 +64.19,201,4.8,28af1eaa77c8 +64.304,201,4.9,28af1eaa77c8 +64.413,201,4.1,28af1eaa77c8 +64.523,201,5.2,28af1eaa77c8 +64.633,201,6.7,28af1eaa77c8 +64.743,201,7.0,28af1eaa77c8 +64.858,201,6.8,28af1eaa77c8 +64.965,201,7.3,28af1eaa77c8 +65.083,201,7.2,f3f2f7801b56 +65.191,201,7.3,f3f2f7801b56 +65.302,201,5.4,f3f2f7801b56 +65.407,201,5.4,28af1eaa77c8 +65.513,201,5.9,28af1eaa77c8 +65.627,201,7.4,28af1eaa77c8 +65.742,201,7.4,f3f2f7801b56 +65.854,201,6.4,f3f2f7801b56 +65.97,201,6.4,28af1eaa77c8 +66.084,201,7.2,28af1eaa77c8 +66.191,201,5.9,f3f2f7801b56 +66.304,201,6.3,28af1eaa77c8 +66.412,201,6.3,28af1eaa77c8 +66.528,201,6.5,28af1eaa77c8 +66.644,201,6.3,f3f2f7801b56 +66.76,201,4.3,28af1eaa77c8 +66.867,201,4.7,f3f2f7801b56 +66.975,201,5.2,f3f2f7801b56 +67.082,201,5.6,f3f2f7801b56 +67.195,201,5.8,f3f2f7801b56 +67.304,201,7.2,f3f2f7801b56 +67.422,201,6.0,28af1eaa77c8 +67.538,201,9.6,f3f2f7801b56 +67.657,201,6.3,28af1eaa77c8 +67.774,201,5.1,28af1eaa77c8 +67.886,201,6.3,28af1eaa77c8 +68.003,201,6.4,f3f2f7801b56 +68.111,201,6.8,f3f2f7801b56 +68.227,201,7.1,f3f2f7801b56 +68.344,201,6.8,28af1eaa77c8 +68.453,201,8.1,28af1eaa77c8 +68.571,201,10.0,f3f2f7801b56 +68.686,201,7.8,f3f2f7801b56 +68.802,201,8.0,28af1eaa77c8 +68.918,201,8.8,f3f2f7801b56 +69.029,201,7.0,f3f2f7801b56 +69.139,201,6.7,f3f2f7801b56 +69.249,201,7.4,f3f2f7801b56 +69.357,201,6.3,28af1eaa77c8 +69.47,201,6.0,f3f2f7801b56 +69.583,201,4.6,f3f2f7801b56 +69.688,201,7.2,28af1eaa77c8 +69.799,201,6.9,f3f2f7801b56 +69.906,201,6.6,28af1eaa77c8 +70.018,201,6.0,28af1eaa77c8 +70.132,201,6.6,f3f2f7801b56 +70.242,201,5.8,f3f2f7801b56 +70.35,201,7.8,f3f2f7801b56 +70.461,201,7.8,f3f2f7801b56 +70.578,201,9.4,28af1eaa77c8 +70.692,201,8.6,28af1eaa77c8 +70.808,201,6.4,28af1eaa77c8 +70.924,201,6.6,f3f2f7801b56 +71.033,201,6.9,f3f2f7801b56 +71.148,201,6.7,28af1eaa77c8 +71.26,201,7.0,28af1eaa77c8 +71.375,201,6.4,28af1eaa77c8 +71.489,201,7.3,f3f2f7801b56 +71.607,201,6.7,f3f2f7801b56 +71.722,201,6.1,28af1eaa77c8 +71.832,201,5.6,28af1eaa77c8 +71.941,201,4.9,28af1eaa77c8 +72.056,201,4.8,28af1eaa77c8 +72.171,201,6.0,f3f2f7801b56 +72.278,201,6.4,f3f2f7801b56 +72.391,201,8.2,f3f2f7801b56 +72.507,201,8.7,f3f2f7801b56 +72.625,201,7.8,28af1eaa77c8 +72.741,201,7.3,f3f2f7801b56 +72.856,201,7.3,f3f2f7801b56 +72.966,201,7.4,f3f2f7801b56 +73.083,201,5.9,28af1eaa77c8 +73.191,201,7.4,28af1eaa77c8 +73.309,201,7.4,f3f2f7801b56 +73.425,201,6.6,f3f2f7801b56 +73.536,201,7.2,f3f2f7801b56 +73.652,201,7.7,28af1eaa77c8 +73.767,201,7.9,28af1eaa77c8 +73.883,201,7.6,f3f2f7801b56 +73.991,201,7.8,f3f2f7801b56 +74.108,201,7.5,28af1eaa77c8 +74.216,201,7.8,f3f2f7801b56 +74.325,201,7.5,f3f2f7801b56 +74.441,201,7.8,28af1eaa77c8 +74.559,201,6.1,f3f2f7801b56 +74.671,201,4.1,f3f2f7801b56 +74.777,201,4.9,28af1eaa77c8 +74.882,201,5.4,28af1eaa77c8 +74.997,201,7.0,28af1eaa77c8 +75.113,201,14.4,28af1eaa77c8 +75.229,201,14.8,f3f2f7801b56 +75.353,201,9.4,28af1eaa77c8 +75.468,201,5.5,f3f2f7801b56 diff --git a/evidence/rolling-from-8d28d45/result.json b/evidence/rolling-from-8d28d45/result.json new file mode 100644 index 00000000..e9e2776d --- /dev/null +++ b/evidence/rolling-from-8d28d45/result.json @@ -0,0 +1,55 @@ +{ + "label": "action", + "action": [ + "scripts/rolling-update.sh" + ], + "action_exit": 0, + "action_seconds": 7.695, + "action_finished_before_load_end": true, + "lb_up_before": 2, + "lb_up_min": 2, + "detection_seconds": null, + "rejoin_seconds": null, + "probe": { + "attempts": 897, + "failed_attempts": 0, + "first_failure_s": null, + "recovery_s": 0.0, + "instances_before": { + "00548d9b229e": 116, + "7a4c320f9b13": 113 + }, + "instances_during_outage": {}, + "instances_after": { + "f3f2f7801b56": 305, + "28af1eaa77c8": 298 + } + }, + "lb_counters": { + "wretr": 2, + "wredis": 54, + "econ": 0, + "eresp": 2, + "hrsp_5xx": 0 + }, + "k6": { + "requests": 9000, + "rps": 100.0, + "ok_rps": 100.0, + "failed_ratio": 0, + "dropped_iterations": 1, + "p50_ms": 2.44, + "p95_ms": 3.4, + "p99_ms": 5.19, + "exit": 0, + "rate": 100 + }, + "checks": { + "action_ok": true, + "load_errors_ok": true, + "write_recovery_ok": true, + "capacity_kept": true, + "all_replaced": true + }, + "pass": true +} diff --git a/evidence/rolling/action.txt b/evidence/rolling/action.txt new file mode 100644 index 00000000..695127bb --- /dev/null +++ b/evidence/rolling/action.txt @@ -0,0 +1,29 @@ +17:03:06 migrate once before the first replacement +{"reason":"up to date","event":"migration.completed","level":"info","ts":"2026-10-28T17:03:07.198467Z"} + Container mtusi2026lab3-api-3 Running + Container mtusi2026lab3-api-4 Running + Container mtusi2026lab3-api-5 Creating + Container mtusi2026lab3-api-5 Created + Container mtusi2026lab3-api-5 Starting + Container mtusi2026lab3-api-5 Started + Container mtusi2026lab3-api-5 Waiting + Container mtusi2026lab3-api-4 Waiting + Container mtusi2026lab3-api-3 Waiting + Container mtusi2026lab3-api-5 Healthy + Container mtusi2026lab3-api-4 Healthy + Container mtusi2026lab3-api-3 Healthy +17:03:09 replace 28af1eaa77c8 image=13e52dc26774 -> 239278da12b2 image=9d621745fe6b + Container mtusi2026lab3-api-4 Running + Container mtusi2026lab3-api-5 Running + Container mtusi2026lab3-api-6 Creating + Container mtusi2026lab3-api-6 Created + Container mtusi2026lab3-api-6 Starting + Container mtusi2026lab3-api-6 Started + Container mtusi2026lab3-api-6 Waiting + Container mtusi2026lab3-api-5 Waiting + Container mtusi2026lab3-api-4 Waiting + Container mtusi2026lab3-api-6 Healthy + Container mtusi2026lab3-api-5 Healthy + Container mtusi2026lab3-api-4 Healthy +17:03:15 replace f3f2f7801b56 image=13e52dc26774 -> 58b4bb97f5db image=9d621745fe6b +17:03:18 done: 2 replicas replaced diff --git a/evidence/rolling/k6/k6.txt b/evidence/rolling/k6/k6.txt new file mode 100644 index 00000000..861de260 --- /dev/null +++ b/evidence/rolling/k6/k6.txt @@ -0,0 +1,2 @@ +time="2026-10-28T17:02:41Z" level=info msg=LOAD-START source=console +measure: 9001 requests, failed ratio 0 diff --git a/evidence/rolling/k6/summary.json b/evidence/rolling/k6/summary.json new file mode 100644 index 00000000..a00b3f89 --- /dev/null +++ b/evidence/rolling/k6/summary.json @@ -0,0 +1,309 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "message 201", + "path": "::message 201", + "id": "6445aff337e1ade0bc10402fbac79882", + "passes": 8011, + "fails": 0 + }, + { + "name": "history 200", + "path": "::history 200", + "id": "fe27b3b8edd292c7f2170a6357b087d0", + "passes": 1991, + "fails": 0 + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "p(90)", + "p(95)", + "p(99)", + "max" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": false, + "isStdErrTTY": false, + "testRunDurationMs": 100517.995699 + }, + "metrics": { + "http_req_duration{scenario:measure,name:post_message}": { + "contains": "time", + "values": { + "min": 1.047, + "med": 2.479375, + "p(90)": 3.189667, + "p(95)": 3.528, + "p(99)": 5.010616199999999, + "max": 34.016209, + "avg": 2.5117758226007587 + }, + "type": "trend" + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 10052 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 3.46374535, + "p(99)": 5.3474348299999965, + "max": 37.816292, + "avg": 2.467581968165534, + "min": 0.921375, + "med": 2.3897294999999996, + "p(90)": 3.1171042 + } + }, + "dropped_iterations": { + "contains": "default", + "values": { + "count": 0, + "rate": 0 + }, + "type": "counter" + }, + "http_req_duration{scenario:measure,name:get_history}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 2.393597505617978, + "min": 1.0205, + "med": 2.3533955, + "p(90)": 3.094084, + "p(95)": 3.4177839499999996, + "p(99)": 4.954887389999999, + "max": 34.899875 + } + }, + "http_req_duration{scenario:measure}": { + "type": "trend", + "contains": "time", + "values": { + "min": 1.0205, + "med": 2.446709, + "p(90)": 3.171334, + "p(95)": 3.494916, + "p(99)": 5.012416, + "max": 34.899875, + "avg": 2.48840537440285 + } + }, + "iterations": { + "contains": "default", + "values": { + "count": 10002, + "rate": 99.50457060396306 + }, + "type": "counter" + }, + "iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 9001, + "rate": 89.54615476967321 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "p(99)": 0.09522858, + "max": 0.414042, + "avg": 0.03010926104257849, + "min": 0.00975, + "med": 0.026917, + "p(90)": 0.042287800000000014, + "p(95)": 0.0535 + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 2.4403125, + "p(90)": 3.1764834, + "p(95)": 3.5353623499999984, + "p(99)": 5.448676919999996, + "max": 37.870792, + "avg": 2.515697025964987, + "min": 0.941792 + } + }, + "iteration_duration": { + "values": { + "p(90)": 3.2874497, + "p(95)": 3.6300624999999993, + "p(99)": 5.31617759, + "max": 34.996916, + "avg": 2.583033877824437, + "min": 0.980167, + "med": 2.5414795 + }, + "type": "trend", + "contains": "time" + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 20, + "min": 20, + "max": 20 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "p(99)": 0, + "max": 0.318417, + "avg": 0.0003617108038201353, + "min": 0, + "med": 0, + "p(90)": 0 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "p(99)": 0, + "max": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.01800579675686432, + "min": 0.002708, + "med": 0.016041, + "p(90)": 0.021667, + "p(95)": 0.028875, + "p(99)": 0.07724749999999998, + "max": 0.4805 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 10002, + "fails": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "max": 1, + "value": 0, + "min": 0 + } + }, + "dropped_iterations{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 0, + "rate": 0 + } + }, + "http_req_blocked": { + "contains": "time", + "values": { + "max": 0.609875, + "avg": 0.0027873373458017765, + "min": 0.000667, + "med": 0.001917, + "p(90)": 0.002958, + "p(95)": 0.0037084499999999994, + "p(99)": 0.013042 + }, + "type": "trend" + }, + "http_reqs{scenario:measure}": { + "type": "counter", + "contains": "default", + "values": { + "count": 9001, + "rate": 89.54615476967321 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 10052, + "rate": 100.00199397230921 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "med": 2.4403125, + "p(90)": 3.1764834, + "p(95)": 3.5353623499999984, + "p(99)": 5.448676919999996, + "max": 37.870792, + "avg": 2.515697025964987, + "min": 0.941792 + } + }, + "data_sent": { + "values": { + "count": 2929568, + "rate": 29144.711647181648 + }, + "type": "counter", + "contains": "data" + }, + "http_req_failed{scenario:measure}": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 9001 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 24805536, + "rate": 246777.0654150317 + } + } + } +} \ No newline at end of file diff --git a/evidence/rolling/lb_up.csv b/evidence/rolling/lb_up.csv new file mode 100644 index 00000000..0a509836 --- /dev/null +++ b/evidence/rolling/lb_up.csv @@ -0,0 +1,914 @@ +t_rel_s,up +-25.809,2 +-25.702,2 +-25.597,2 +-25.493,2 +-25.388,2 +-25.274,2 +-25.166,2 +-25.055,2 +-24.945,2 +-24.83,2 +-24.718,2 +-24.602,2 +-24.489,2 +-24.385,2 +-24.272,2 +-24.156,2 +-24.047,2 +-23.933,2 +-23.823,2 +-23.707,2 +-23.596,2 +-23.481,2 +-23.367,2 +-23.256,2 +-23.148,2 +-23.045,2 +-22.93,2 +-22.815,2 +-22.708,2 +-22.595,2 +-22.481,2 +-22.367,2 +-22.253,2 +-22.138,2 +-22.031,2 +-21.92,2 +-21.814,2 +-21.698,2 +-21.584,2 +-21.467,2 +-21.351,2 +-21.237,2 +-21.12,2 +-21.013,2 +-20.899,2 +-20.789,2 +-20.677,2 +-20.562,2 +-20.452,2 +-20.34,2 +-20.233,2 +-20.121,2 +-20.005,2 +-19.891,2 +-19.781,2 +-19.67,2 +-19.559,2 +-19.445,2 +-19.332,2 +-19.223,2 +-19.105,2 +-18.994,2 +-18.881,2 +-18.765,2 +-18.651,2 +-18.541,2 +-18.432,2 +-18.318,2 +-18.211,2 +-18.096,2 +-17.983,2 +-17.877,2 +-17.767,2 +-17.655,2 +-17.544,2 +-17.431,2 +-17.314,2 +-17.206,2 +-17.091,2 +-16.981,2 +-16.865,2 +-16.757,2 +-16.643,2 +-16.537,2 +-16.418,2 +-16.306,2 +-16.2,2 +-16.086,2 +-15.973,2 +-15.864,2 +-15.748,2 +-15.64,2 +-15.524,2 +-15.409,2 +-15.301,2 +-15.194,2 +-15.083,2 +-14.968,2 +-14.853,2 +-14.744,2 +-14.637,2 +-14.522,2 +-14.412,2 +-14.297,2 +-14.192,2 +-14.077,2 +-13.972,2 +-13.868,2 +-13.763,2 +-13.654,2 +-13.544,2 +-13.439,2 +-13.329,2 +-13.223,2 +-13.115,2 +-13.006,2 +-12.9,2 +-12.789,2 +-12.674,2 +-12.56,2 +-12.449,2 +-12.339,2 +-12.231,2 +-12.115,2 +-12.008,2 +-11.897,2 +-11.791,2 +-11.677,2 +-11.565,2 +-11.45,2 +-11.338,2 +-11.223,2 +-11.116,2 +-11.001,2 +-10.896,2 +-10.782,2 +-10.673,2 +-10.562,2 +-10.456,2 +-10.344,2 +-10.237,2 +-10.125,2 +-10.014,2 +-9.906,2 +-9.8,2 +-9.69,2 +-9.58,2 +-9.468,2 +-9.356,2 +-9.245,2 +-9.137,2 +-9.027,2 +-8.916,2 +-8.809,2 +-8.696,2 +-8.588,2 +-8.472,2 +-8.356,2 +-8.248,2 +-8.139,2 +-8.027,2 +-7.916,2 +-7.801,2 +-7.683,2 +-7.571,2 +-7.456,2 +-7.339,2 +-7.231,2 +-7.115,2 +-6.999,2 +-6.89,2 +-6.781,2 +-6.668,2 +-6.554,2 +-6.447,2 +-6.336,2 +-6.22,2 +-6.1,2 +-5.989,2 +-5.871,2 +-5.757,2 +-5.645,2 +-5.531,2 +-5.416,2 +-5.307,2 +-5.192,2 +-5.077,2 +-4.97,2 +-4.863,2 +-4.752,2 +-4.636,2 +-4.523,2 +-4.408,2 +-4.299,2 +-4.189,2 +-4.076,2 +-3.964,2 +-3.857,2 +-3.742,2 +-3.626,2 +-3.514,2 +-3.398,2 +-3.29,2 +-3.177,2 +-3.066,2 +-2.96,2 +-2.844,2 +-2.731,2 +-2.625,2 +-2.518,2 +-2.405,2 +-2.289,2 +-2.181,2 +-2.065,2 +-1.953,2 +-1.845,2 +-1.734,2 +-1.624,2 +-1.517,2 +-1.403,2 +-1.287,2 +-1.172,2 +-1.056,2 +-0.944,2 +-0.828,2 +-0.721,2 +-0.607,2 +-0.491,2 +-0.381,2 +-0.264,2 +-0.156,2 +-0.044,2 +0.064,2 +0.173,2 +0.281,2 +0.388,2 +0.493,2 +0.598,2 +0.702,2 +0.814,2 +0.918,2 +1.033,2 +1.137,2 +1.242,2 +1.355,2 +1.466,2 +1.577,2 +1.692,2 +1.805,2 +1.911,2 +2.025,2 +2.135,2 +2.244,2 +2.358,2 +2.462,2 +2.567,2 +2.68,2 +2.79,2 +2.9,2 +3.004,2 +3.113,2 +3.22,3 +3.328,3 +3.438,3 +3.542,3 +3.654,3 +3.757,3 +3.863,3 +3.969,3 +4.077,3 +4.184,3 +4.294,3 +4.408,2 +4.514,2 +4.628,2 +4.739,2 +4.85,2 +4.966,2 +5.077,2 +5.194,2 +5.297,2 +5.413,2 +5.518,2 +5.634,2 +5.741,2 +5.844,2 +5.956,2 +6.059,2 +6.165,2 +6.269,2 +6.373,2 +6.487,2 +6.599,2 +6.71,2 +6.825,2 +6.937,2 +7.048,2 +7.152,2 +7.256,2 +7.364,2 +7.477,2 +7.585,2 +7.694,2 +7.802,2 +7.916,2 +8.028,2 +8.138,2 +8.246,2 +8.352,2 +8.461,2 +8.573,2 +8.68,2 +8.795,2 +8.91,3 +9.025,3 +9.132,3 +9.241,3 +9.347,3 +9.456,3 +9.572,3 +9.682,3 +9.786,3 +9.901,3 +10.006,3 +10.111,3 +10.219,3 +10.327,2 +10.443,2 +10.556,2 +10.665,2 +10.779,2 +10.886,2 +10.994,2 +11.104,2 +11.219,2 +11.326,2 +11.437,2 +11.542,2 +11.655,2 +11.759,2 +11.863,2 +11.968,2 +12.073,2 +12.187,2 +12.302,2 +12.409,2 +12.515,2 +12.631,2 +12.744,2 +12.856,2 +12.961,2 +13.076,2 +13.19,2 +13.307,2 +13.422,2 +13.537,2 +13.646,2 +13.752,2 +13.861,2 +13.969,2 +14.078,2 +14.186,2 +14.301,2 +14.417,2 +14.531,2 +14.645,2 +14.752,2 +14.861,2 +14.977,2 +15.088,2 +15.204,2 +15.311,2 +15.418,2 +15.527,2 +15.635,2 +15.751,2 +15.863,2 +15.975,2 +16.087,2 +16.202,2 +16.315,2 +16.426,2 +16.537,2 +16.651,2 +16.765,2 +16.881,2 +16.994,2 +17.106,2 +17.22,2 +17.334,2 +17.445,2 +17.56,2 +17.669,2 +17.778,2 +17.891,2 +17.996,2 +18.108,2 +18.219,2 +18.335,2 +18.448,2 +18.557,2 +18.663,2 +18.77,2 +18.886,2 +18.996,2 +19.109,2 +19.224,2 +19.337,2 +19.447,2 +19.554,2 +19.661,2 +19.769,2 +19.885,2 +19.995,2 +20.101,2 +20.216,2 +20.327,2 +20.435,2 +20.544,2 +20.652,2 +20.762,2 +20.869,2 +20.976,2 +21.084,2 +21.199,2 +21.305,2 +21.419,2 +21.531,2 +21.647,2 +21.761,2 +21.869,2 +21.983,2 +22.097,2 +22.211,2 +22.326,2 +22.439,2 +22.555,2 +22.67,2 +22.777,2 +22.89,2 +22.997,2 +23.109,2 +23.215,2 +23.327,2 +23.437,2 +23.55,2 +23.661,2 +23.769,2 +23.879,2 +23.994,2 +24.105,2 +24.219,2 +24.332,2 +24.437,2 +24.552,2 +24.66,2 +24.77,2 +24.883,2 +24.996,2 +25.103,2 +25.22,2 +25.335,2 +25.448,2 +25.561,2 +25.673,2 +25.787,2 +25.893,2 +26.007,2 +26.124,2 +26.239,2 +26.353,2 +26.469,2 +26.584,2 +26.698,2 +26.811,2 +26.925,2 +27.035,2 +27.146,2 +27.254,2 +27.37,2 +27.488,2 +27.602,2 +27.716,2 +27.821,2 +27.935,2 +28.044,2 +28.154,2 +28.266,2 +28.373,2 +28.487,2 +28.602,2 +28.709,2 +28.826,2 +28.941,2 +29.055,2 +29.161,2 +29.269,2 +29.383,2 +29.494,2 +29.606,2 +29.718,2 +29.832,2 +29.941,2 +30.055,2 +30.169,2 +30.282,2 +30.388,2 +30.503,2 +30.607,2 +30.716,2 +30.821,2 +30.936,2 +31.048,2 +31.164,2 +31.278,2 +31.383,2 +31.491,2 +31.596,2 +31.703,2 +31.809,2 +31.921,2 +32.037,2 +32.143,2 +32.253,2 +32.363,2 +32.472,2 +32.582,2 +32.691,2 +32.796,2 +32.903,2 +33.011,2 +33.125,2 +33.24,2 +33.353,2 +33.466,2 +33.578,2 +33.689,2 +33.804,2 +33.913,2 +34.026,2 +34.131,2 +34.247,2 +34.353,2 +34.459,2 +34.574,2 +34.686,2 +34.796,2 +34.911,2 +35.025,2 +35.134,2 +35.244,2 +35.358,2 +35.47,2 +35.577,2 +35.685,2 +35.798,2 +35.909,2 +36.021,2 +36.134,2 +36.241,2 +36.356,2 +36.467,2 +36.578,2 +36.687,2 +36.796,2 +36.911,2 +37.024,2 +37.138,2 +37.25,2 +37.366,2 +37.481,2 +37.597,2 +37.704,2 +37.815,2 +37.928,2 +38.035,2 +38.149,2 +38.264,2 +38.38,2 +38.494,2 +38.609,2 +38.718,2 +38.833,2 +38.946,2 +39.056,2 +39.167,2 +39.281,2 +39.39,2 +39.498,2 +39.614,2 +39.723,2 +39.839,2 +39.952,2 +40.066,2 +40.178,2 +40.293,2 +40.404,2 +40.511,2 +40.619,2 +40.736,2 +40.848,2 +40.961,2 +41.066,2 +41.17,2 +41.276,2 +41.384,2 +41.491,2 +41.596,2 +41.703,2 +41.813,2 +41.917,2 +42.022,2 +42.137,2 +42.24,2 +42.352,2 +42.464,2 +42.573,2 +42.681,2 +42.794,2 +42.91,2 +43.028,2 +43.142,2 +43.252,2 +43.358,2 +43.473,2 +43.585,2 +43.696,2 +43.81,2 +43.924,2 +44.031,2 +44.136,2 +44.251,2 +44.367,2 +44.483,2 +44.595,2 +44.699,2 +44.811,2 +44.919,2 +45.028,2 +45.135,2 +45.244,2 +45.359,2 +45.465,2 +45.58,2 +45.695,2 +45.806,2 +45.922,2 +46.036,2 +46.151,2 +46.263,2 +46.369,2 +46.477,2 +46.592,2 +46.703,2 +46.818,2 +46.933,2 +47.049,2 +47.165,2 +47.28,2 +47.395,2 +47.508,2 +47.621,2 +47.736,2 +47.85,2 +47.961,2 +48.077,2 +48.189,2 +48.305,2 +48.411,2 +48.519,2 +48.631,2 +48.744,2 +48.856,2 +48.97,2 +49.077,2 +49.193,2 +49.298,2 +49.403,2 +49.509,2 +49.615,2 +49.723,2 +49.834,2 +49.943,2 +50.058,2 +50.171,2 +50.284,2 +50.395,2 +50.504,2 +50.611,2 +50.719,2 +50.832,2 +50.943,2 +51.056,2 +51.165,2 +51.271,2 +51.386,2 +51.498,2 +51.614,2 +51.725,2 +51.829,2 +51.936,2 +52.05,2 +52.161,2 +52.276,2 +52.379,2 +52.486,2 +52.6,2 +52.715,2 +52.828,2 +52.943,2 +53.049,2 +53.152,2 +53.264,2 +53.367,2 +53.472,2 +53.58,2 +53.691,2 +53.803,2 +53.914,2 +54.023,2 +54.134,2 +54.243,2 +54.356,2 +54.467,2 +54.575,2 +54.685,2 +54.791,2 +54.905,2 +55.019,2 +55.134,2 +55.245,2 +55.361,2 +55.476,2 +55.592,2 +55.706,2 +55.82,2 +55.935,2 +56.044,2 +56.157,2 +56.273,2 +56.388,2 +56.502,2 +56.61,2 +56.723,2 +56.831,2 +56.94,2 +57.044,2 +57.15,2 +57.262,2 +57.37,2 +57.479,2 +57.582,2 +57.693,2 +57.807,2 +57.923,2 +58.037,2 +58.143,2 +58.257,2 +58.366,2 +58.476,2 +58.582,2 +58.69,2 +58.793,2 +58.898,2 +59.011,2 +59.121,2 +59.224,2 +59.333,2 +59.445,2 +59.561,2 +59.672,2 +59.786,2 +59.894,2 +60.009,2 +60.118,2 +60.228,2 +60.336,2 +60.445,2 +60.553,2 +60.669,2 +60.778,2 +60.891,2 +61.001,2 +61.117,2 +61.226,2 +61.337,2 +61.448,2 +61.56,2 +61.674,2 +61.789,2 +61.904,2 +62.018,2 +62.134,2 +62.246,2 +62.358,2 +62.465,2 +62.577,2 +62.682,2 +62.794,2 +62.903,2 +63.012,2 +63.127,2 +63.241,2 +63.355,2 +63.465,2 +63.577,2 +63.692,2 +63.803,2 +63.906,2 +64.02,2 +64.126,2 +64.241,2 +64.351,2 +64.465,2 +64.577,2 +64.684,2 +64.79,2 +64.903,2 +65.011,2 +65.121,2 +65.226,2 +65.333,2 +65.447,2 +65.56,2 +65.667,2 +65.78,2 +65.894,2 +66.002,2 +66.109,2 +66.225,2 +66.332,2 +66.448,2 +66.558,2 +66.67,2 +66.778,2 +66.892,2 +67.005,2 +67.113,2 +67.224,2 +67.335,2 +67.448,2 +67.555,2 +67.661,2 +67.771,2 +67.875,2 +67.986,2 +68.094,2 +68.21,2 +68.324,2 +68.436,2 +68.542,2 +68.656,2 +68.772,2 +68.884,2 +68.997,2 +69.112,2 +69.226,2 +69.338,2 +69.453,2 +69.569,2 +69.675,2 +69.783,2 +69.891,2 +69.996,2 +70.111,2 +70.22,2 +70.328,2 +70.435,2 +70.55,2 +70.661,2 +70.777,2 +70.892,2 +71.007,2 +71.12,2 +71.228,2 +71.335,2 +71.446,2 +71.561,2 +71.676,2 +71.789,2 +71.895,2 +72.005,2 +72.113,2 +72.219,2 +72.332,2 +72.446,2 +72.553,2 +72.661,2 +72.777,2 +72.885,2 +72.996,2 +73.105,2 +73.22,2 +73.332,2 +73.438,2 +73.554,2 +73.666,2 +73.77,2 +73.887,2 +74.0,2 +74.109,2 +74.216,2 +74.327,2 +74.43,2 +74.535,2 +74.642,2 +74.754,2 +74.867,2 +74.979,2 +75.094,2 +75.208,2 +75.322,2 +75.437,2 diff --git a/evidence/rolling/probe.csv b/evidence/rolling/probe.csv new file mode 100644 index 00000000..41d7e496 --- /dev/null +++ b/evidence/rolling/probe.csv @@ -0,0 +1,905 @@ +t_rel_s,status,latency_ms,instance +-25.809,201,7.0,f3f2f7801b56 +-25.702,201,6.6,28af1eaa77c8 +-25.595,201,3.8,f3f2f7801b56 +-25.491,201,4.3,28af1eaa77c8 +-25.386,201,3.7,28af1eaa77c8 +-25.273,201,5.3,28af1eaa77c8 +-25.161,201,4.3,f3f2f7801b56 +-25.047,201,4.0,28af1eaa77c8 +-24.941,201,4.6,28af1eaa77c8 +-24.831,201,5.3,28af1eaa77c8 +-24.718,201,7.1,28af1eaa77c8 +-24.61,201,8.0,28af1eaa77c8 +-24.5,201,8.7,28af1eaa77c8 +-24.389,201,9.0,28af1eaa77c8 +-24.275,201,9.8,f3f2f7801b56 +-24.165,201,6.6,f3f2f7801b56 +-24.049,201,6.7,f3f2f7801b56 +-23.932,201,4.0,28af1eaa77c8 +-23.822,201,6.4,28af1eaa77c8 +-23.714,201,7.5,28af1eaa77c8 +-23.598,201,8.5,28af1eaa77c8 +-23.489,201,6.4,28af1eaa77c8 +-23.382,201,7.3,28af1eaa77c8 +-23.269,201,6.9,28af1eaa77c8 +-23.152,201,6.4,f3f2f7801b56 +-23.042,201,6.2,f3f2f7801b56 +-22.927,201,6.7,f3f2f7801b56 +-22.812,201,5.6,28af1eaa77c8 +-22.706,201,6.5,f3f2f7801b56 +-22.595,201,8.5,28af1eaa77c8 +-22.478,201,5.5,28af1eaa77c8 +-22.365,201,4.8,28af1eaa77c8 +-22.25,201,4.8,f3f2f7801b56 +-22.139,201,6.7,f3f2f7801b56 +-22.025,201,5.4,28af1eaa77c8 +-21.915,201,5.4,f3f2f7801b56 +-21.806,201,8.8,28af1eaa77c8 +-21.692,201,6.8,28af1eaa77c8 +-21.578,201,8.3,28af1eaa77c8 +-21.465,201,8.0,f3f2f7801b56 +-21.355,201,8.7,f3f2f7801b56 +-21.237,201,8.6,f3f2f7801b56 +-21.119,201,6.8,28af1eaa77c8 +-21.005,201,7.7,28af1eaa77c8 +-20.891,201,6.3,f3f2f7801b56 +-20.778,201,5.8,f3f2f7801b56 +-20.67,201,6.9,f3f2f7801b56 +-20.553,201,7.0,28af1eaa77c8 +-20.444,201,6.2,28af1eaa77c8 +-20.336,201,5.6,f3f2f7801b56 +-20.229,201,7.0,f3f2f7801b56 +-20.121,201,6.9,f3f2f7801b56 +-20.008,201,8.8,f3f2f7801b56 +-19.899,201,6.9,f3f2f7801b56 +-19.782,201,7.0,28af1eaa77c8 +-19.673,201,6.7,28af1eaa77c8 +-19.56,201,7.1,28af1eaa77c8 +-19.444,201,7.6,f3f2f7801b56 +-19.332,201,8.7,f3f2f7801b56 +-19.214,201,7.9,28af1eaa77c8 +-19.096,201,6.9,28af1eaa77c8 +-18.988,201,7.5,28af1eaa77c8 +-18.873,201,6.5,f3f2f7801b56 +-18.763,201,8.1,f3f2f7801b56 +-18.652,201,9.4,f3f2f7801b56 +-18.538,201,6.9,f3f2f7801b56 +-18.423,201,4.8,28af1eaa77c8 +-18.316,201,5.6,f3f2f7801b56 +-18.209,201,5.7,f3f2f7801b56 +-18.093,201,5.3,28af1eaa77c8 +-17.979,201,5.5,28af1eaa77c8 +-17.873,201,7.6,28af1eaa77c8 +-17.763,201,8.7,28af1eaa77c8 +-17.649,201,5.9,28af1eaa77c8 +-17.539,201,6.6,28af1eaa77c8 +-17.43,201,8.2,28af1eaa77c8 +-17.316,201,5.3,28af1eaa77c8 +-17.208,201,7.3,28af1eaa77c8 +-17.097,201,6.1,28af1eaa77c8 +-16.981,201,8.4,f3f2f7801b56 +-16.865,201,8.9,28af1eaa77c8 +-16.755,201,6.5,f3f2f7801b56 +-16.644,201,9.3,28af1eaa77c8 +-16.524,201,4.6,f3f2f7801b56 +-16.414,201,4.2,28af1eaa77c8 +-16.303,201,4.9,f3f2f7801b56 +-16.191,201,7.6,f3f2f7801b56 +-16.075,201,5.6,f3f2f7801b56 +-15.968,201,5.2,f3f2f7801b56 +-15.86,201,6.2,f3f2f7801b56 +-15.752,201,8.7,f3f2f7801b56 +-15.639,201,13.1,f3f2f7801b56 +-15.519,201,5.9,28af1eaa77c8 +-15.41,201,5.7,28af1eaa77c8 +-15.301,201,6.5,28af1eaa77c8 +-15.191,201,7.2,28af1eaa77c8 +-15.082,201,6.3,28af1eaa77c8 +-14.968,201,7.2,f3f2f7801b56 +-14.86,201,6.0,f3f2f7801b56 +-14.744,201,7.3,28af1eaa77c8 +-14.628,201,3.5,28af1eaa77c8 +-14.523,201,8.4,28af1eaa77c8 +-14.411,201,8.6,28af1eaa77c8 +-14.295,201,4.1,28af1eaa77c8 +-14.189,201,6.4,28af1eaa77c8 +-14.073,201,3.9,28af1eaa77c8 +-13.969,201,4.5,28af1eaa77c8 +-13.856,201,3.5,28af1eaa77c8 +-13.748,201,4.8,28af1eaa77c8 +-13.641,201,5.3,28af1eaa77c8 +-13.529,201,5.7,28af1eaa77c8 +-13.414,201,6.9,28af1eaa77c8 +-13.297,201,6.7,f3f2f7801b56 +-13.182,201,7.2,28af1eaa77c8 +-13.075,201,5.1,f3f2f7801b56 +-12.967,201,7.0,f3f2f7801b56 +-12.851,201,7.8,28af1eaa77c8 +-12.742,201,7.9,28af1eaa77c8 +-12.631,201,7.9,28af1eaa77c8 +-12.513,201,6.5,28af1eaa77c8 +-12.397,201,6.8,f3f2f7801b56 +-12.289,201,6.5,f3f2f7801b56 +-12.182,201,4.9,28af1eaa77c8 +-12.074,201,5.0,28af1eaa77c8 +-11.961,201,5.4,f3f2f7801b56 +-11.848,201,5.2,f3f2f7801b56 +-11.739,201,7.2,f3f2f7801b56 +-11.63,201,8.3,f3f2f7801b56 +-11.512,201,7.5,28af1eaa77c8 +-11.394,201,6.6,28af1eaa77c8 +-11.281,201,6.6,f3f2f7801b56 +-11.167,201,7.1,f3f2f7801b56 +-11.051,201,8.0,28af1eaa77c8 +-10.937,201,6.4,28af1eaa77c8 +-10.83,201,8.1,28af1eaa77c8 +-10.714,201,6.7,28af1eaa77c8 +-10.604,201,8.9,28af1eaa77c8 +-10.489,201,9.6,f3f2f7801b56 +-10.374,201,6.7,f3f2f7801b56 +-10.265,201,6.8,f3f2f7801b56 +-10.155,201,6.3,f3f2f7801b56 +-10.045,201,6.7,f3f2f7801b56 +-9.928,201,5.8,28af1eaa77c8 +-9.812,201,6.6,f3f2f7801b56 +-9.697,201,4.9,f3f2f7801b56 +-9.589,201,4.7,f3f2f7801b56 +-9.483,201,5.9,f3f2f7801b56 +-9.372,201,6.6,f3f2f7801b56 +-9.255,201,7.4,f3f2f7801b56 +-9.139,201,7.7,28af1eaa77c8 +-9.031,201,7.8,28af1eaa77c8 +-8.914,201,7.2,28af1eaa77c8 +-8.805,201,6.3,28af1eaa77c8 +-8.692,201,7.1,28af1eaa77c8 +-8.584,201,6.6,28af1eaa77c8 +-8.472,201,7.9,f3f2f7801b56 +-8.356,201,7.9,f3f2f7801b56 +-8.241,201,7.8,28af1eaa77c8 +-8.125,201,6.8,28af1eaa77c8 +-8.016,201,6.6,28af1eaa77c8 +-7.903,201,7.3,28af1eaa77c8 +-7.787,201,5.6,f3f2f7801b56 +-7.678,201,7.0,f3f2f7801b56 +-7.563,201,6.3,f3f2f7801b56 +-7.455,201,10.2,f3f2f7801b56 +-7.344,201,7.1,f3f2f7801b56 +-7.228,201,5.9,28af1eaa77c8 +-7.121,201,9.3,28af1eaa77c8 +-7.006,201,5.0,28af1eaa77c8 +-6.891,201,6.7,f3f2f7801b56 +-6.783,201,6.5,f3f2f7801b56 +-6.667,201,6.9,f3f2f7801b56 +-6.552,201,5.8,f3f2f7801b56 +-6.446,201,5.6,f3f2f7801b56 +-6.332,201,7.3,28af1eaa77c8 +-6.22,201,12.9,28af1eaa77c8 +-6.099,201,9.4,f3f2f7801b56 +-5.989,201,8.4,f3f2f7801b56 +-5.873,201,7.9,28af1eaa77c8 +-5.764,201,9.6,28af1eaa77c8 +-5.649,201,8.8,28af1eaa77c8 +-5.53,201,8.2,f3f2f7801b56 +-5.414,201,7.4,f3f2f7801b56 +-5.297,201,5.9,28af1eaa77c8 +-5.181,201,6.2,f3f2f7801b56 +-5.066,201,7.3,f3f2f7801b56 +-4.956,201,6.5,f3f2f7801b56 +-4.844,201,7.3,f3f2f7801b56 +-4.736,201,6.2,f3f2f7801b56 +-4.624,201,6.1,f3f2f7801b56 +-4.513,201,4.4,f3f2f7801b56 +-4.406,201,3.5,f3f2f7801b56 +-4.299,201,6.5,f3f2f7801b56 +-4.187,201,6.2,f3f2f7801b56 +-4.073,201,7.2,28af1eaa77c8 +-3.955,201,7.0,28af1eaa77c8 +-3.838,201,5.9,f3f2f7801b56 +-3.731,201,5.7,f3f2f7801b56 +-3.617,201,5.5,f3f2f7801b56 +-3.505,201,7.0,f3f2f7801b56 +-3.398,201,7.3,f3f2f7801b56 +-3.29,201,8.2,f3f2f7801b56 +-3.174,201,6.6,f3f2f7801b56 +-3.066,201,8.2,f3f2f7801b56 +-2.956,201,7.1,f3f2f7801b56 +-2.844,201,8.7,f3f2f7801b56 +-2.725,201,5.9,28af1eaa77c8 +-2.619,201,5.2,28af1eaa77c8 +-2.505,201,6.1,28af1eaa77c8 +-2.389,201,7.1,f3f2f7801b56 +-2.281,201,7.0,f3f2f7801b56 +-2.164,201,7.3,f3f2f7801b56 +-2.048,201,6.9,28af1eaa77c8 +-1.935,201,6.3,28af1eaa77c8 +-1.828,201,4.0,28af1eaa77c8 +-1.719,201,4.6,28af1eaa77c8 +-1.608,201,4.3,28af1eaa77c8 +-1.493,201,5.8,28af1eaa77c8 +-1.38,201,7.0,f3f2f7801b56 +-1.272,201,7.4,f3f2f7801b56 +-1.156,201,5.9,f3f2f7801b56 +-1.047,201,6.7,f3f2f7801b56 +-0.939,201,8.1,f3f2f7801b56 +-0.823,201,7.6,f3f2f7801b56 +-0.711,201,6.2,28af1eaa77c8 +-0.602,201,6.5,f3f2f7801b56 +-0.493,201,9.7,28af1eaa77c8 +-0.375,201,6.7,28af1eaa77c8 +-0.266,201,7.5,28af1eaa77c8 +-0.151,201,7.1,f3f2f7801b56 +-0.039,201,7.7,f3f2f7801b56 +0.077,201,5.5,f3f2f7801b56 +0.186,201,4.2,f3f2f7801b56 +0.292,201,4.9,f3f2f7801b56 +0.404,201,4.5,f3f2f7801b56 +0.51,201,4.7,f3f2f7801b56 +0.625,201,5.1,f3f2f7801b56 +0.736,201,4.8,f3f2f7801b56 +0.846,201,4.3,f3f2f7801b56 +0.955,201,5.3,f3f2f7801b56 +1.061,201,4.5,f3f2f7801b56 +1.166,201,4.3,28af1eaa77c8 +1.272,201,4.5,28af1eaa77c8 +1.386,201,4.6,28af1eaa77c8 +1.501,201,4.1,f3f2f7801b56 +1.615,201,4.5,f3f2f7801b56 +1.726,201,6.3,f3f2f7801b56 +1.84,201,5.5,28af1eaa77c8 +1.954,201,5.4,28af1eaa77c8 +2.069,201,7.2,f3f2f7801b56 +2.183,201,7.6,f3f2f7801b56 +2.294,201,5.2,f3f2f7801b56 +2.405,201,4.6,f3f2f7801b56 +2.51,201,5.7,f3f2f7801b56 +2.626,201,11.0,28af1eaa77c8 +2.739,201,8.6,28af1eaa77c8 +2.852,201,5.3,28af1eaa77c8 +2.965,201,6.5,28af1eaa77c8 +3.073,201,6.9,28af1eaa77c8 +3.184,201,8.0,28af1eaa77c8 +3.302,201,5.8,f3f2f7801b56 +3.413,201,4.9,f3f2f7801b56 +3.518,201,4.6,28af1eaa77c8 +3.628,201,6.0,f3f2f7801b56 +3.741,201,5.2,f3f2f7801b56 +3.853,201,6.6,f3f2f7801b56 +3.968,201,8.7,239278da12b2 +4.085,201,7.3,239278da12b2 +4.202,201,6.3,28af1eaa77c8 +4.319,201,6.7,f3f2f7801b56 +4.434,201,6.5,239278da12b2 +4.551,201,6.4,f3f2f7801b56 +4.662,201,6.1,f3f2f7801b56 +4.769,201,7.3,f3f2f7801b56 +4.885,201,7.2,f3f2f7801b56 +4.994,201,8.1,f3f2f7801b56 +5.106,201,7.0,f3f2f7801b56 +5.219,201,7.1,f3f2f7801b56 +5.33,201,7.0,239278da12b2 +5.444,201,7.2,239278da12b2 +5.551,201,6.7,239278da12b2 +5.665,201,5.1,239278da12b2 +5.778,201,5.5,f3f2f7801b56 +5.885,201,6.0,239278da12b2 +6.001,201,4.9,f3f2f7801b56 +6.11,201,5.3,f3f2f7801b56 +6.218,201,4.5,239278da12b2 +6.325,201,5.1,239278da12b2 +6.44,201,4.0,f3f2f7801b56 +6.553,201,5.2,f3f2f7801b56 +6.659,201,5.3,f3f2f7801b56 +6.769,201,5.9,f3f2f7801b56 +6.885,201,7.1,f3f2f7801b56 +6.994,201,6.0,f3f2f7801b56 +7.11,201,6.0,239278da12b2 +7.219,201,6.6,239278da12b2 +7.334,201,6.9,239278da12b2 +7.449,201,5.2,f3f2f7801b56 +7.554,201,7.4,239278da12b2 +7.666,201,6.1,239278da12b2 +7.782,201,7.7,f3f2f7801b56 +7.894,201,6.1,f3f2f7801b56 +8.004,201,5.2,f3f2f7801b56 +8.11,201,6.5,f3f2f7801b56 +8.225,201,7.6,f3f2f7801b56 +8.343,201,7.7,239278da12b2 +8.456,201,9.6,f3f2f7801b56 +8.569,201,6.9,239278da12b2 +8.676,201,10.4,f3f2f7801b56 +8.794,201,9.3,f3f2f7801b56 +8.913,201,5.0,239278da12b2 +9.019,201,6.4,239278da12b2 +9.13,201,5.4,239278da12b2 +9.239,201,6.5,239278da12b2 +9.351,201,5.5,239278da12b2 +9.466,201,6.4,239278da12b2 +9.576,201,7.8,239278da12b2 +9.686,201,5.4,239278da12b2 +9.796,201,7.3,239278da12b2 +9.912,201,4.9,f3f2f7801b56 +10.027,201,6.6,f3f2f7801b56 +10.136,201,6.1,f3f2f7801b56 +10.246,201,7.3,f3f2f7801b56 +10.361,201,7.0,58b4bb97f5db +10.472,201,7.0,58b4bb97f5db +10.59,201,8.7,239278da12b2 +10.708,201,5.6,58b4bb97f5db +10.816,201,5.5,239278da12b2 +10.929,201,6.1,58b4bb97f5db +11.036,201,5.9,239278da12b2 +11.144,201,4.9,239278da12b2 +11.249,201,6.4,239278da12b2 +11.361,201,5.7,239278da12b2 +11.472,201,5.2,239278da12b2 +11.581,201,4.4,239278da12b2 +11.687,201,3.9,58b4bb97f5db +11.794,201,3.8,58b4bb97f5db +11.904,201,4.3,58b4bb97f5db +12.012,201,6.2,58b4bb97f5db +12.119,201,7.7,58b4bb97f5db +12.228,201,8.6,239278da12b2 +12.343,201,6.8,58b4bb97f5db +12.456,201,6.5,58b4bb97f5db +12.569,201,4.9,239278da12b2 +12.684,201,4.0,239278da12b2 +12.798,201,5.6,58b4bb97f5db +12.911,201,5.2,58b4bb97f5db +13.019,201,7.8,58b4bb97f5db +13.134,201,7.3,58b4bb97f5db +13.242,201,5.4,58b4bb97f5db +13.352,201,6.6,58b4bb97f5db +13.469,201,7.2,239278da12b2 +13.578,201,7.2,239278da12b2 +13.695,201,6.9,239278da12b2 +13.812,201,7.4,58b4bb97f5db +13.926,201,6.7,58b4bb97f5db +14.033,201,7.5,58b4bb97f5db +14.147,201,8.3,58b4bb97f5db +14.265,201,7.1,239278da12b2 +14.376,201,6.9,239278da12b2 +14.488,201,7.2,58b4bb97f5db +14.602,201,6.0,58b4bb97f5db +14.71,201,8.3,58b4bb97f5db +14.819,201,8.1,58b4bb97f5db +14.937,201,7.6,239278da12b2 +15.053,201,7.8,239278da12b2 +15.169,201,8.6,58b4bb97f5db +15.28,201,7.5,58b4bb97f5db +15.39,201,6.6,58b4bb97f5db +15.497,201,7.0,239278da12b2 +15.611,201,8.4,58b4bb97f5db +15.729,201,9.4,239278da12b2 +15.846,201,8.6,239278da12b2 +15.962,201,7.5,58b4bb97f5db +16.077,201,7.4,239278da12b2 +16.186,201,8.3,58b4bb97f5db +16.296,201,7.8,58b4bb97f5db +16.407,201,7.5,58b4bb97f5db +16.515,201,6.3,58b4bb97f5db +16.622,201,5.8,58b4bb97f5db +16.732,201,7.4,58b4bb97f5db +16.848,201,7.7,239278da12b2 +16.966,201,7.8,239278da12b2 +17.077,201,7.3,239278da12b2 +17.188,201,6.6,58b4bb97f5db +17.305,201,7.5,58b4bb97f5db +17.422,201,8.2,239278da12b2 +17.538,201,8.5,58b4bb97f5db +17.652,201,7.3,58b4bb97f5db +17.768,201,4.9,58b4bb97f5db +17.881,201,4.2,239278da12b2 +17.994,201,4.9,239278da12b2 +18.109,201,8.3,58b4bb97f5db +18.219,201,8.8,58b4bb97f5db +18.334,201,7.8,58b4bb97f5db +18.449,201,8.8,239278da12b2 +18.559,201,6.6,239278da12b2 +18.669,201,6.1,239278da12b2 +18.781,201,5.9,239278da12b2 +18.897,201,6.1,239278da12b2 +19.011,201,7.2,58b4bb97f5db +19.128,201,7.3,239278da12b2 +19.243,201,6.6,239278da12b2 +19.356,201,8.3,239278da12b2 +19.474,201,8.0,58b4bb97f5db +19.584,201,7.4,58b4bb97f5db +19.696,201,6.1,58b4bb97f5db +19.805,201,7.6,58b4bb97f5db +19.916,201,6.7,58b4bb97f5db +20.03,201,7.9,239278da12b2 +20.144,201,8.0,239278da12b2 +20.261,201,6.9,58b4bb97f5db +20.378,201,7.9,239278da12b2 +20.486,201,7.3,58b4bb97f5db +20.597,201,8.4,239278da12b2 +20.716,201,5.6,239278da12b2 +20.825,201,7.1,239278da12b2 +20.938,201,6.8,58b4bb97f5db +21.052,201,6.6,58b4bb97f5db +21.161,201,7.9,58b4bb97f5db +21.269,201,7.9,58b4bb97f5db +21.386,201,7.3,58b4bb97f5db +21.503,201,7.6,239278da12b2 +21.619,201,8.2,58b4bb97f5db +21.727,201,4.8,239278da12b2 +21.842,201,5.8,58b4bb97f5db +21.954,201,6.4,58b4bb97f5db +22.068,201,6.4,58b4bb97f5db +22.176,201,7.2,58b4bb97f5db +22.285,201,7.3,58b4bb97f5db +22.394,201,8.1,58b4bb97f5db +22.511,201,10.3,239278da12b2 +22.629,201,8.0,58b4bb97f5db +22.744,201,6.9,58b4bb97f5db +22.853,201,5.1,58b4bb97f5db +22.965,201,3.4,58b4bb97f5db +23.074,201,5.0,58b4bb97f5db +23.189,201,5.4,239278da12b2 +23.296,201,6.9,58b4bb97f5db +23.411,201,7.6,239278da12b2 +23.527,201,7.3,239278da12b2 +23.636,201,7.0,239278da12b2 +23.745,201,7.2,239278da12b2 +23.86,201,7.4,58b4bb97f5db +23.977,201,8.2,239278da12b2 +24.086,201,8.2,58b4bb97f5db +24.203,201,7.9,239278da12b2 +24.311,201,7.6,239278da12b2 +24.427,201,9.6,58b4bb97f5db +24.538,201,7.5,58b4bb97f5db +24.652,201,8.1,58b4bb97f5db +24.761,201,4.9,58b4bb97f5db +24.874,201,6.5,58b4bb97f5db +24.986,201,7.6,58b4bb97f5db +25.104,201,8.4,239278da12b2 +25.219,201,9.5,58b4bb97f5db +25.33,201,7.0,58b4bb97f5db +25.446,201,9.1,239278da12b2 +25.565,201,5.9,239278da12b2 +25.676,201,6.5,239278da12b2 +25.79,201,5.2,58b4bb97f5db +25.901,201,6.7,58b4bb97f5db +26.011,201,7.1,58b4bb97f5db +26.127,201,6.0,58b4bb97f5db +26.241,201,7.0,239278da12b2 +26.356,201,7.1,239278da12b2 +26.464,201,7.4,239278da12b2 +26.574,201,5.7,239278da12b2 +26.686,201,8.9,239278da12b2 +26.802,201,6.2,58b4bb97f5db +26.912,201,6.1,58b4bb97f5db +27.028,201,8.1,58b4bb97f5db +27.146,201,8.8,58b4bb97f5db +27.265,201,7.2,58b4bb97f5db +27.375,201,7.7,58b4bb97f5db +27.484,201,9.4,58b4bb97f5db +27.594,201,8.1,58b4bb97f5db +27.711,201,7.5,239278da12b2 +27.828,201,6.4,58b4bb97f5db +27.943,201,7.0,58b4bb97f5db +28.06,201,3.8,239278da12b2 +28.172,201,4.4,239278da12b2 +28.278,201,5.3,58b4bb97f5db +28.393,201,5.2,239278da12b2 +28.502,201,5.7,239278da12b2 +28.613,201,6.3,239278da12b2 +28.722,201,7.5,239278da12b2 +28.838,201,5.6,239278da12b2 +28.95,201,5.7,58b4bb97f5db +29.066,201,6.0,58b4bb97f5db +29.178,201,7.1,58b4bb97f5db +29.293,201,6.7,239278da12b2 +29.405,201,7.6,239278da12b2 +29.519,201,6.1,58b4bb97f5db +29.628,201,8.2,58b4bb97f5db +29.746,201,7.2,58b4bb97f5db +29.862,201,7.6,239278da12b2 +29.976,201,6.6,239278da12b2 +30.092,201,7.8,58b4bb97f5db +30.204,201,6.6,58b4bb97f5db +30.32,201,7.2,239278da12b2 +30.435,201,5.7,239278da12b2 +30.544,201,4.6,239278da12b2 +30.656,201,5.7,239278da12b2 +30.769,201,6.7,58b4bb97f5db +30.886,201,5.7,58b4bb97f5db +30.992,201,4.6,58b4bb97f5db +31.098,201,7.1,58b4bb97f5db +31.206,201,7.1,239278da12b2 +31.322,201,4.6,58b4bb97f5db +31.43,201,5.9,58b4bb97f5db +31.537,201,6.5,239278da12b2 +31.644,201,7.1,239278da12b2 +31.761,201,7.0,58b4bb97f5db +31.877,201,6.7,58b4bb97f5db +31.994,201,7.4,239278da12b2 +32.109,201,7.9,58b4bb97f5db +32.219,201,7.8,58b4bb97f5db +32.336,201,7.3,58b4bb97f5db +32.443,201,7.2,58b4bb97f5db +32.561,201,8.5,239278da12b2 +32.674,201,8.5,239278da12b2 +32.787,201,8.1,239278da12b2 +32.902,201,8.7,58b4bb97f5db +33.021,201,5.7,239278da12b2 +33.137,201,5.8,239278da12b2 +33.244,201,4.6,239278da12b2 +33.359,201,5.6,58b4bb97f5db +33.466,201,7.4,58b4bb97f5db +33.574,201,6.9,239278da12b2 +33.688,201,8.7,58b4bb97f5db +33.802,201,6.1,58b4bb97f5db +33.912,201,5.1,58b4bb97f5db +34.027,201,6.7,58b4bb97f5db +34.142,201,4.2,239278da12b2 +34.248,201,6.0,239278da12b2 +34.356,201,4.6,58b4bb97f5db +34.469,201,6.6,239278da12b2 +34.577,201,7.4,239278da12b2 +34.694,201,5.9,239278da12b2 +34.805,201,5.0,239278da12b2 +34.92,201,7.7,58b4bb97f5db +35.038,201,6.0,58b4bb97f5db +35.151,201,6.2,239278da12b2 +35.265,201,7.2,239278da12b2 +35.382,201,5.5,58b4bb97f5db +35.498,201,7.3,239278da12b2 +35.611,201,6.6,239278da12b2 +35.723,201,6.6,239278da12b2 +35.835,201,6.5,239278da12b2 +35.949,201,6.8,58b4bb97f5db +36.06,201,9.2,58b4bb97f5db +36.178,201,6.2,239278da12b2 +36.286,201,5.6,58b4bb97f5db +36.396,201,6.4,58b4bb97f5db +36.503,201,6.4,58b4bb97f5db +36.619,201,8.9,239278da12b2 +36.736,201,6.7,239278da12b2 +36.853,201,8.4,58b4bb97f5db +36.969,201,7.4,239278da12b2 +37.086,201,6.0,239278da12b2 +37.192,201,6.5,239278da12b2 +37.309,201,7.5,58b4bb97f5db +37.424,201,8.0,58b4bb97f5db +37.539,201,8.1,239278da12b2 +37.65,201,7.9,239278da12b2 +37.764,201,6.8,239278da12b2 +37.873,201,6.9,239278da12b2 +37.983,201,8.5,239278da12b2 +38.092,201,6.1,239278da12b2 +38.203,201,7.8,239278da12b2 +38.312,201,6.8,239278da12b2 +38.423,201,4.2,239278da12b2 +38.531,201,3.3,239278da12b2 +38.644,201,4.8,239278da12b2 +38.756,201,6.5,239278da12b2 +38.869,201,8.0,58b4bb97f5db +38.986,201,8.7,58b4bb97f5db +39.105,201,5.8,239278da12b2 +39.211,201,6.5,239278da12b2 +39.325,201,6.8,239278da12b2 +39.439,201,6.7,58b4bb97f5db +39.553,201,6.7,58b4bb97f5db +39.66,201,4.8,58b4bb97f5db +39.773,201,6.1,58b4bb97f5db +39.885,201,7.2,58b4bb97f5db +39.993,201,4.5,58b4bb97f5db +40.098,201,7.5,58b4bb97f5db +40.215,201,6.8,58b4bb97f5db +40.332,201,7.1,239278da12b2 +40.449,201,8.6,58b4bb97f5db +40.561,201,8.3,58b4bb97f5db +40.677,201,7.6,239278da12b2 +40.794,201,6.2,239278da12b2 +40.91,201,7.7,58b4bb97f5db +41.021,201,5.7,239278da12b2 +41.133,201,3.9,58b4bb97f5db +41.241,201,5.7,58b4bb97f5db +41.357,201,6.0,58b4bb97f5db +41.467,201,7.5,58b4bb97f5db +41.583,201,8.6,239278da12b2 +41.695,201,6.9,239278da12b2 +41.802,201,5.8,239278da12b2 +41.912,201,5.0,239278da12b2 +42.023,201,5.0,239278da12b2 +42.133,201,6.0,239278da12b2 +42.245,201,5.9,239278da12b2 +42.352,201,7.0,239278da12b2 +42.467,201,6.0,239278da12b2 +42.574,201,8.9,239278da12b2 +42.685,201,5.3,239278da12b2 +42.801,201,6.7,58b4bb97f5db +42.916,201,7.5,58b4bb97f5db +43.031,201,9.6,239278da12b2 +43.142,201,6.9,239278da12b2 +43.258,201,6.0,58b4bb97f5db +43.364,201,5.7,239278da12b2 +43.474,201,6.7,239278da12b2 +43.587,201,4.5,239278da12b2 +43.699,201,4.8,58b4bb97f5db +43.81,201,7.4,58b4bb97f5db +43.924,201,7.4,58b4bb97f5db +44.034,201,4.8,58b4bb97f5db +44.144,201,5.6,58b4bb97f5db +44.26,201,7.8,239278da12b2 +44.378,201,7.2,58b4bb97f5db +44.494,201,4.2,58b4bb97f5db +44.601,201,4.8,58b4bb97f5db +44.714,201,6.3,58b4bb97f5db +44.828,201,6.7,58b4bb97f5db +44.944,201,6.6,239278da12b2 +45.056,201,6.3,239278da12b2 +45.17,201,5.9,58b4bb97f5db +45.283,201,5.1,58b4bb97f5db +45.397,201,6.9,58b4bb97f5db +45.505,201,7.6,58b4bb97f5db +45.613,201,5.3,58b4bb97f5db +45.728,201,7.8,239278da12b2 +45.837,201,7.3,58b4bb97f5db +45.952,201,7.6,239278da12b2 +46.068,201,7.2,58b4bb97f5db +46.185,201,7.7,58b4bb97f5db +46.3,201,7.2,239278da12b2 +46.417,201,6.9,239278da12b2 +46.533,201,6.9,58b4bb97f5db +46.647,201,5.9,58b4bb97f5db +46.761,201,7.6,239278da12b2 +46.869,201,7.0,239278da12b2 +46.982,201,7.0,239278da12b2 +47.09,201,8.4,239278da12b2 +47.207,201,6.2,239278da12b2 +47.319,201,5.9,58b4bb97f5db +47.432,201,6.6,58b4bb97f5db +47.546,201,6.5,58b4bb97f5db +47.661,201,7.6,239278da12b2 +47.776,201,6.7,239278da12b2 +47.892,201,7.9,58b4bb97f5db +48.003,201,6.7,58b4bb97f5db +48.111,201,7.0,58b4bb97f5db +48.223,201,6.8,58b4bb97f5db +48.334,201,6.2,58b4bb97f5db +48.449,201,7.6,239278da12b2 +48.567,201,7.9,239278da12b2 +48.685,201,8.7,58b4bb97f5db +48.803,201,4.4,239278da12b2 +48.917,201,5.7,58b4bb97f5db +49.03,201,6.2,58b4bb97f5db +49.145,201,7.0,58b4bb97f5db +49.26,201,8.2,239278da12b2 +49.369,201,7.2,239278da12b2 +49.478,201,7.7,239278da12b2 +49.586,201,8.9,58b4bb97f5db +49.698,201,7.6,239278da12b2 +49.808,201,6.9,239278da12b2 +49.925,201,6.7,239278da12b2 +50.041,201,6.9,58b4bb97f5db +50.156,201,8.2,58b4bb97f5db +50.274,201,7.5,239278da12b2 +50.389,201,8.3,58b4bb97f5db +50.507,201,6.1,58b4bb97f5db +50.615,201,5.3,58b4bb97f5db +50.73,201,7.2,239278da12b2 +50.844,201,5.2,239278da12b2 +50.956,201,6.7,239278da12b2 +51.071,201,6.7,58b4bb97f5db +51.182,201,6.7,58b4bb97f5db +51.295,201,6.0,58b4bb97f5db +51.403,201,4.3,58b4bb97f5db +51.507,201,3.7,239278da12b2 +51.621,201,5.7,58b4bb97f5db +51.727,201,4.2,239278da12b2 +51.831,201,4.6,239278da12b2 +51.936,201,5.4,58b4bb97f5db +52.052,201,4.9,239278da12b2 +52.161,201,6.0,239278da12b2 +52.269,201,5.3,239278da12b2 +52.383,201,4.9,239278da12b2 +52.494,201,4.9,239278da12b2 +52.601,201,6.9,239278da12b2 +52.714,201,6.6,239278da12b2 +52.828,201,8.8,58b4bb97f5db +52.947,201,4.7,58b4bb97f5db +53.055,201,3.7,58b4bb97f5db +53.16,201,4.3,58b4bb97f5db +53.268,201,4.0,239278da12b2 +53.373,201,3.1,239278da12b2 +53.476,201,3.7,58b4bb97f5db +53.584,201,6.0,58b4bb97f5db +53.691,201,6.3,58b4bb97f5db +53.808,201,4.3,58b4bb97f5db +53.913,201,7.7,58b4bb97f5db +54.025,201,4.4,58b4bb97f5db +54.133,201,7.4,58b4bb97f5db +54.247,201,7.9,58b4bb97f5db +54.361,201,5.1,239278da12b2 +54.467,201,5.1,58b4bb97f5db +54.577,201,5.5,58b4bb97f5db +54.688,201,6.2,58b4bb97f5db +54.804,201,7.9,239278da12b2 +54.921,201,6.2,58b4bb97f5db +55.031,201,6.8,58b4bb97f5db +55.144,201,6.4,58b4bb97f5db +55.253,201,6.5,58b4bb97f5db +55.361,201,8.7,58b4bb97f5db +55.477,201,6.6,58b4bb97f5db +55.593,201,5.6,239278da12b2 +55.703,201,8.4,239278da12b2 +55.818,201,7.7,58b4bb97f5db +55.935,201,8.8,58b4bb97f5db +56.052,201,7.4,239278da12b2 +56.16,201,6.3,239278da12b2 +56.273,201,7.4,239278da12b2 +56.389,201,8.7,58b4bb97f5db +56.507,201,6.8,58b4bb97f5db +56.62,201,4.4,239278da12b2 +56.734,201,3.4,239278da12b2 +56.839,201,4.4,239278da12b2 +56.944,201,9.2,58b4bb97f5db +57.053,201,4.4,58b4bb97f5db +57.159,201,3.5,58b4bb97f5db +57.265,201,5.0,239278da12b2 +57.38,201,4.1,58b4bb97f5db +57.494,201,4.1,58b4bb97f5db +57.602,201,4.0,58b4bb97f5db +57.711,201,5.5,58b4bb97f5db +57.827,201,6.4,58b4bb97f5db +57.938,201,5.7,58b4bb97f5db +58.052,201,6.9,239278da12b2 +58.161,201,4.4,239278da12b2 +58.268,201,3.6,58b4bb97f5db +58.372,201,3.9,58b4bb97f5db +58.478,201,2.9,239278da12b2 +58.581,201,5.2,239278da12b2 +58.689,201,5.0,239278da12b2 +58.794,201,5.0,58b4bb97f5db +58.9,201,3.5,58b4bb97f5db +59.011,201,5.1,58b4bb97f5db +59.117,201,6.5,58b4bb97f5db +59.224,201,6.3,239278da12b2 +59.339,201,6.8,58b4bb97f5db +59.453,201,6.7,58b4bb97f5db +59.561,201,6.7,58b4bb97f5db +59.677,201,5.6,58b4bb97f5db +59.793,201,6.1,239278da12b2 +59.907,201,6.3,239278da12b2 +60.019,201,6.3,58b4bb97f5db +60.136,201,7.0,58b4bb97f5db +60.253,201,7.0,239278da12b2 +60.369,201,8.6,58b4bb97f5db +60.488,201,7.0,239278da12b2 +60.603,201,7.5,239278da12b2 +60.72,201,7.9,58b4bb97f5db +60.828,201,7.4,58b4bb97f5db +60.941,201,8.0,58b4bb97f5db +61.054,201,6.7,58b4bb97f5db +61.169,201,7.5,239278da12b2 +61.287,201,8.2,239278da12b2 +61.397,201,8.0,58b4bb97f5db +61.514,201,5.8,58b4bb97f5db +61.624,201,6.3,58b4bb97f5db +61.739,201,6.8,239278da12b2 +61.855,201,6.4,239278da12b2 +61.962,201,6.6,239278da12b2 +62.069,201,7.6,239278da12b2 +62.18,201,6.7,239278da12b2 +62.295,201,6.4,239278da12b2 +62.406,201,5.2,239278da12b2 +62.515,201,5.9,239278da12b2 +62.631,201,6.3,58b4bb97f5db +62.747,201,7.9,58b4bb97f5db +62.865,201,7.3,239278da12b2 +62.982,201,7.2,58b4bb97f5db +63.096,201,9.0,239278da12b2 +63.211,201,7.5,239278da12b2 +63.326,201,6.2,239278da12b2 +63.442,201,5.2,58b4bb97f5db +63.549,201,7.3,58b4bb97f5db +63.664,201,5.4,58b4bb97f5db +63.774,201,4.3,58b4bb97f5db +63.885,201,4.9,58b4bb97f5db +63.994,201,6.3,58b4bb97f5db +64.109,201,6.1,239278da12b2 +64.216,201,3.8,58b4bb97f5db +64.328,201,5.3,58b4bb97f5db +64.437,201,6.5,239278da12b2 +64.55,201,6.6,239278da12b2 +64.666,201,7.0,239278da12b2 +64.781,201,9.0,58b4bb97f5db +64.9,201,6.6,239278da12b2 +65.011,201,7.3,239278da12b2 +65.128,201,6.6,239278da12b2 +65.236,201,6.9,239278da12b2 +65.353,201,5.0,58b4bb97f5db +65.459,201,7.4,58b4bb97f5db +65.568,201,7.1,58b4bb97f5db +65.682,201,5.4,58b4bb97f5db +65.789,201,5.5,58b4bb97f5db +65.904,201,6.4,58b4bb97f5db +66.012,201,6.9,58b4bb97f5db +66.126,201,6.9,58b4bb97f5db +66.234,201,7.3,58b4bb97f5db +66.343,201,6.2,58b4bb97f5db +66.454,201,5.5,58b4bb97f5db +66.56,201,7.0,58b4bb97f5db +66.669,201,8.2,58b4bb97f5db +66.783,201,7.5,58b4bb97f5db +66.899,201,5.4,58b4bb97f5db +67.011,201,3.7,239278da12b2 +67.116,201,6.2,58b4bb97f5db +67.224,201,7.7,58b4bb97f5db +67.335,201,8.0,58b4bb97f5db +67.445,201,6.2,58b4bb97f5db +67.552,201,7.7,58b4bb97f5db +67.667,201,4.8,58b4bb97f5db +67.781,201,5.7,239278da12b2 +67.894,201,6.1,239278da12b2 +68.008,201,7.9,58b4bb97f5db +68.122,201,6.6,58b4bb97f5db +68.229,201,7.4,58b4bb97f5db +68.345,201,7.7,58b4bb97f5db +68.456,201,6.1,58b4bb97f5db +68.569,201,7.5,239278da12b2 +68.683,201,7.7,239278da12b2 +68.791,201,6.8,239278da12b2 +68.908,201,6.6,58b4bb97f5db +69.016,201,6.8,239278da12b2 +69.125,201,5.7,239278da12b2 +69.239,201,6.9,58b4bb97f5db +69.348,201,4.5,239278da12b2 +69.461,201,4.2,58b4bb97f5db +69.57,201,5.4,58b4bb97f5db +69.676,201,5.8,239278da12b2 +69.783,201,6.2,239278da12b2 +69.894,201,6.1,239278da12b2 +70.003,201,5.9,239278da12b2 +70.111,201,8.4,239278da12b2 +70.227,201,7.5,239278da12b2 +70.336,201,6.1,239278da12b2 +70.446,201,6.6,239278da12b2 +70.561,201,6.4,58b4bb97f5db +70.677,201,7.1,58b4bb97f5db +70.793,201,6.9,239278da12b2 +70.905,201,5.9,239278da12b2 +71.013,201,5.4,239278da12b2 +71.12,201,8.3,239278da12b2 +71.236,201,6.5,239278da12b2 +71.345,201,6.4,239278da12b2 +71.459,201,7.1,58b4bb97f5db +71.573,201,6.3,58b4bb97f5db +71.682,201,6.6,58b4bb97f5db +71.799,201,4.1,58b4bb97f5db +71.911,201,5.3,239278da12b2 +72.018,201,7.7,239278da12b2 +72.127,201,6.8,58b4bb97f5db +72.236,201,7.9,58b4bb97f5db +72.346,201,7.1,58b4bb97f5db +72.455,201,7.4,58b4bb97f5db +72.57,201,6.8,239278da12b2 +72.678,201,7.8,239278da12b2 +72.794,201,6.9,239278da12b2 +72.911,201,6.9,58b4bb97f5db +73.022,201,6.8,58b4bb97f5db +73.135,201,6.6,58b4bb97f5db +73.245,201,8.2,58b4bb97f5db +73.363,201,6.6,239278da12b2 +73.476,201,6.3,239278da12b2 +73.586,201,6.9,239278da12b2 +73.703,201,7.4,58b4bb97f5db +73.819,201,6.4,239278da12b2 +73.935,201,6.3,239278da12b2 +74.048,201,4.7,239278da12b2 +74.162,201,4.9,58b4bb97f5db +74.272,201,6.3,58b4bb97f5db +74.388,201,7.1,58b4bb97f5db +74.496,201,4.3,58b4bb97f5db +74.604,201,4.4,58b4bb97f5db +74.713,201,4.7,58b4bb97f5db +74.826,201,5.7,58b4bb97f5db +74.938,201,5.2,239278da12b2 +75.053,201,6.2,58b4bb97f5db +75.169,201,7.1,239278da12b2 +75.286,201,7.1,58b4bb97f5db +75.393,201,5.5,239278da12b2 +75.5,201,7.4,58b4bb97f5db diff --git a/evidence/rolling/result.json b/evidence/rolling/result.json new file mode 100644 index 00000000..bf7dd127 --- /dev/null +++ b/evidence/rolling/result.json @@ -0,0 +1,55 @@ +{ + "label": "action", + "action": [ + "scripts/rolling-update.sh" + ], + "action_exit": 0, + "action_seconds": 11.782, + "action_finished_before_load_end": true, + "lb_up_before": 2, + "lb_up_min": 2, + "detection_seconds": null, + "rejoin_seconds": null, + "probe": { + "attempts": 904, + "failed_attempts": 0, + "first_failure_s": null, + "recovery_s": 0.0, + "instances_before": { + "f3f2f7801b56": 114, + "28af1eaa77c8": 117 + }, + "instances_during_outage": {}, + "instances_after": { + "58b4bb97f5db": 314, + "239278da12b2": 254 + } + }, + "lb_counters": { + "wretr": 0, + "wredis": 0, + "econ": 0, + "eresp": 0, + "hrsp_5xx": 0 + }, + "k6": { + "requests": 9001, + "rps": 100.01, + "ok_rps": 100.01, + "failed_ratio": 0, + "dropped_iterations": 0, + "p50_ms": 2.45, + "p95_ms": 3.49, + "p99_ms": 5.01, + "exit": 0, + "rate": 100 + }, + "checks": { + "action_ok": true, + "load_errors_ok": true, + "write_recovery_ok": true, + "capacity_kept": true, + "all_replaced": true + }, + "pass": true +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt index c523e2ff..3a41a52f 100644 --- a/evidence/smoke.txt +++ b/evidence/smoke.txt @@ -1,4 +1,4 @@ -date: 2026-10-28T08:47:26Z commit: 0796e29 +date: 2026-10-28T16:53:51Z commit: e00eb6c $ make test BASE_URL=http://localhost:18083 python3 tests/smoke.py --base-url "http://localhost:18083" PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stack.txt b/evidence/stack.txt index 05f1a581..6c7541df 100644 --- a/evidence/stack.txt +++ b/evidence/stack.txt @@ -1,30 +1,31 @@ -date: 2026-10-28T08:48:39Z commit: 0796e29 +date: 2026-10-28T16:53:55Z commit: e00eb6c $ docker compose --env-file .env config --quiet && echo valid valid $ docker compose ps --format '{{.Service}} {{.Name}} {{.Status}} {{.Ports}}' -api mtusi2026lab3-api-1 Up 8 seconds (healthy) -api mtusi2026lab3-api-2 Up 8 seconds (healthy) -lb mtusi2026lab3-lb-1 Up About a minute (healthy) 127.0.0.1:18083->8080/tcp, 127.0.0.1:23083->8404/tcp -postgres mtusi2026lab3-postgres-1 Up About a minute (healthy) -valkey mtusi2026lab3-valkey-1 Up About a minute (healthy) +api mtusi2026lab3-api-1 Up 7 seconds (healthy) +api mtusi2026lab3-api-2 Up 7 seconds (healthy) +lb mtusi2026lab3-lb-1 Up 14 seconds (healthy) 127.0.0.1:18083->8080/tcp, 127.0.0.1:23083->8404/tcp +postgres mtusi2026lab3-postgres-1 Up 14 seconds (healthy) +valkey mtusi2026lab3-valkey-1 Up 14 seconds (healthy) $ docker compose config | grep -c container_name 0 -$ docker compose port api 8080 # реплики не публикуют порт -no port 8080/tcp for container mtusi2026lab3-api-1: +$ docker port <каждая реплика api> # реплики не публикуют порт +a3536e394b3f: [] +27de23496b0f: [] $ docker inspect: hostname = X-Instance-Id, сети, лимиты CPU (nano) и памяти -api 302cfee8f673 /mtusi2026lab3-api-1 nets=mtusi2026lab3_backend, cpus=1000000000 mem=536870912 user=10001:10001 ro=true -api b582bd503bdd /mtusi2026lab3-api-2 nets=mtusi2026lab3_backend, cpus=1000000000 mem=536870912 user=10001:10001 ro=true -lb 10cd61cb6cc5 /mtusi2026lab3-lb-1 nets=mtusi2026lab3_backend,mtusi2026lab3_edge, cpus=1000000000 mem=134217728 user=haproxy ro=true -postgres f06fc06e6824 /mtusi2026lab3-postgres-1 nets=mtusi2026lab3_backend, cpus=2000000000 mem=1073741824 user=70:70 ro=false -valkey 5f8773e8a7ad /mtusi2026lab3-valkey-1 nets=mtusi2026lab3_backend, cpus=500000000 mem=134217728 user=999:1000 ro=false +api 27de23496b0f /mtusi2026lab3-api-2 nets=mtusi2026lab3_backend, cpus=1000000000 mem=536870912 user=10001:10001 ro=true +api a3536e394b3f /mtusi2026lab3-api-1 nets=mtusi2026lab3_backend, cpus=1000000000 mem=536870912 user=10001:10001 ro=true +lb 0b152ec25c09 /mtusi2026lab3-lb-1 nets=mtusi2026lab3_backend,mtusi2026lab3_edge, cpus=1000000000 mem=134217728 user=haproxy ro=true +postgres 67fe9b43b70b /mtusi2026lab3-postgres-1 nets=mtusi2026lab3_backend, cpus=2000000000 mem=1073741824 user=70:70 ro=false +valkey e4c5a748222e /mtusi2026lab3-valkey-1 nets=mtusi2026lab3_backend, cpus=500000000 mem=134217728 user=999:1000 ro=false $ curl LB stats: backend api, server status check api1 UP L7OK 200 api2 UP L7OK 200 BACKEND UP $ for i in 1..6: curl -D - http://127.0.0.1:18083/health/live | X-Instance-Id -x-instance-id: 302cfee8f673 -x-instance-id: b582bd503bdd -x-instance-id: 302cfee8f673 -x-instance-id: b582bd503bdd -x-instance-id: 302cfee8f673 -x-instance-id: b582bd503bdd +x-instance-id: a3536e394b3f +x-instance-id: 27de23496b0f +x-instance-id: a3536e394b3f +x-instance-id: 27de23496b0f +x-instance-id: a3536e394b3f +x-instance-id: 27de23496b0f diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt index 4bb0d685..cca20eca 100644 --- a/evidence/student-tests.txt +++ b/evidence/student-tests.txt @@ -1,14 +1,19 @@ -date: 2026-10-28T08:49:27Z commit: 0796e29 +date: 2026-10-28T17:04:35Z commit: e00eb6c $ make student-test # pytest без маркера stack, по файлам ============================= test session starts ============================== +platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 +rootdir: /Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab3 +configfile: pyproject.toml testpaths: student-tests -collected 249 items / 10 deselected / 239 selected - +plugins: asyncio-1.4.0, anyio-4.15.1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function +collected 252 items / 10 deselected / 242 selected student-tests/api/test_access.py ....... [ 2%] student-tests/api/test_bounded_contexts.py .................. [ 10%] student-tests/api/test_cache.py ..... [ 12%] student-tests/api/test_concurrency.py ... [ 13%] -student-tests/api/test_consistency.py ........ [ 17%] +student-tests/api/test_consistency.py ........ [ 16%] +student-tests/api/test_drain.py .. [ 17%] student-tests/api/test_events.py ....... [ 20%] student-tests/api/test_idempotency.py ............... [ 26%] student-tests/api/test_instance.py .... [ 28%] @@ -17,13 +22,19 @@ student-tests/api/test_negative.py ..................................... [ 45%] [ 45%] student-tests/api/test_operability.py ............. [ 51%] student-tests/api/test_pagination.py ................. [ 58%] -student-tests/api/test_pool.py .. [ 58%] +student-tests/api/test_pool.py .. [ 59%] student-tests/api/test_sessions.py .................. [ 66%] student-tests/api/test_timeout.py ...... [ 69%] student-tests/api/test_wire_format.py .. [ 69%] student-tests/unit/test_authentication.py ..................... [ 78%] student-tests/unit/test_cursor.py ........... [ 83%] -student-tests/unit/test_domain.py ...................................... [ 99%] -.. [100%] +student-tests/unit/test_domain.py ...................................... [ 98%] +.. [ 99%] +student-tests/unit/test_timeouts.py . [100%] +===================== 242 passed, 10 deselected ====================== -===================== 239 passed, 10 deselected in 17.41s ====================== +$ .venv/bin/pytest -s -q student-tests/api/test_drain.py student-tests/unit/test_timeouts.py # SIGTERM-окно и цепочка таймаутов +.окно: ready/live (503, 200), порт закрыт через 1.69 с, выход 2.11 с +запрос, принятый до SIGTERM и дописанный после закрытия порта: HTTP/1.1 201 Created +.обнаружение 1.5 с (зависание 3.5 с), окно 2.0 с, остановка 5.0 с, хук 6.0 с, stop_grace_period 8.0 с; LB server 10.0 с > запрос 8.0 с > БД 3.0 с; readiness 0.8 с < check 1.0 с +. diff --git a/openspec/changes/add-lab3-failover-operations/tasks.md b/openspec/changes/add-lab3-failover-operations/tasks.md index 9b98730f..bfdf8fd2 100644 --- a/openspec/changes/add-lab3-failover-operations/tasks.md +++ b/openspec/changes/add-lab3-failover-operations/tasks.md @@ -26,10 +26,10 @@ - [x] 4.4 `scripts/rolling-update.sh` и `make rolling-update [FROM_REV=…]`: `migrate` один раз, замена с запасом +1, журнал образов - [x] 4.5 Тест отказа PostgreSQL через LB: ответы `503`, live внутри реплики `200`, LB исключает все реплики и сам отвечает `503` - [x] 4.6 CI: `make limits`, `make detection` в acceptance -- [ ] 4.7 Проверка группы: `make up`, `make limits`, `make lb-check`, `make detection`, `make failover-each`, `make rolling-update FROM_REV=8d28d45`, `make test-faults` +- [x] 4.7 Проверка группы: `make up`, `make limits`, `make lb-check`, `make detection`, `make failover-each`, `make rolling-update FROM_REV=8d28d45`, `make test-faults` ## 5. Отчёт -- [ ] 5.1 `evidence/`: detection, failover-each, rolling, limits, lb-check, повторные прогоны уровня «3» на новом коде -- [ ] 5.2 `REPORT.md`: таймауты и обнаружение, строки 4.1–4.4, таблица отказов уровня «4», ограничения -- [ ] 5.3 Проверка группы: `make check`, `make validate`, `make lint`, `make student-test`, `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` для обеих реплик +- [x] 5.1 `evidence/`: detection, failover-each, rolling, limits, lb-check, повторные прогоны уровня «3» на новом коде +- [x] 5.2 `REPORT.md`: таймауты и обнаружение, строки 4.1–4.4, таблица отказов уровня «4», ограничения +- [x] 5.3 Проверка группы: `make check`, `make validate`, `make lint`, `make student-test`, `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` для обеих реплик