diff --git a/REPORT.md b/REPORT.md index bd5137d2..387d2780 100644 --- a/REPORT.md +++ b/REPORT.md @@ -6,8 +6,9 @@ keyset pagination, общий Idempotency-Key, HAProxy и две независ Добавлены приватный S3, вложения и повторяемый reconciler. Публикация и окончательные результаты будут внесены после проверок. -Сейчас завершены реализация и статические проверки; runtime-доказательства -ещё собираются. Этот промежуточный статус не является заявлением PASS. +Публичный smoke и 108 собственных HTTP-проверок прошли на полном стенде +(362.40 s). Проверки отказов/исследование и clean clone ещё выполняются; +окончательная готовность будет указана после их завершения. ## Запуск из чистого clone @@ -18,7 +19,8 @@ MinIO с точным module version скачивается восемью по с ограниченным timeout/числом повторов; затем стандартный `go mod download` проверяет h1 через Go checksum database перед сборкой. Проверки TLS и checksum не отключены. Go parallelism -ограничен двумя потоками, зависимости и компилятор используют BuildKit cache. +задан `GOMAXPROCS=2` и `GOFLAGS=-p=2`; зависимости и компилятор +используют BuildKit cache. ```sh uv sync --locked @@ -103,8 +105,15 @@ endpoint `s3:9000`, region us-east-1, bucket messenger, byte/pixel limits и Readiness проверяет PG и S3. При потере S3 HAProxy может исключить все API и вернуть собственный 503; прямой внутренний API возвращает JSON unavailable. -Таблица наблюдений и ссылки на evidence будут заполнены после выполнения -storage, cluster, HTTP и clean-clone проверок. +| Проверка | Команда | Наблюдение | Доказательство | +| --- | --- | --- | --- | +| SQL migrations | `docker compose run --rm --no-deps migrate`, повтор | 3 migrations, checksum match, repeat idempotent | [migrations-first.json](evidence/migrations-first.json) | +| Публичный API | `make test` | PASS: upload201, Message, original SHA-256 и ACL | [http-first.json](evidence/http-first.json) | +| Прежние возможности и негативные файлы | `uv run pytest student-tests -q` | 108 passed, 362.40 s; в том числе chunked >10 MiB, corrupt/MIME/pixels/filename/foreign chat | [http-first.json](evidence/http-first.json) | +| Закреплённый S3 | `go mod download`, `minio --version`, inspect | module h1 verified; binary 2025-10-15/go1.24.8/arm64; 2 API, реальные CPU/RAM/ports | [minio-source-verification.json](evidence/minio-source-verification.json) | + +Строки storage/cluster/research/clean clone будут добавлены после выполнения +этих проверок. Source HTTP = `f5a7a74`, OCI revision API совпадает с ним. ## Методика исследования diff --git a/evidence/cluster-first.json b/evidence/cluster-first.json new file mode 100644 index 00000000..aca94543 --- /dev/null +++ b/evidence/cluster-first.json @@ -0,0 +1,4458 @@ +{ + "date": "2026-11-15T18:22:35.520155+00:00", + "source_commit": "f5a7a74f679d936d851fa0e58f6cfb59dcd62c50", + "complete": true, + "before": [ + { + "role": "api", + "container": "c9c495521bf2", + "pid": 39030, + "started_at": "2026-11-15T18:03:15.404279669Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "role": "api", + "container": "2af84f3c87dd", + "pid": 80780, + "started_at": "2026-11-15T18:13:52.321424672Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "role": "cache", + "container": "e33ddf35f293", + "pid": 5356, + "started_at": "2026-11-15T17:39:31.281698344Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "lb", + "container": "6d7d02d323ed", + "pid": 39796, + "started_at": "2026-11-15T18:03:26.954206841Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8084" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "a5ffac5a4228", + "pid": 5418, + "started_at": "2026-11-15T17:39:31.614022427Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "c94b86137075", + "pid": 94299, + "started_at": "2026-11-15T18:20:05.941649137Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "s3", + "container": "acc08930aff2", + "pid": 82901, + "started_at": "2026-11-15T18:14:38.360606722Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + } + ], + "parallel_replay": { + "requests": 30, + "messages": 1, + "instances": [ + "ba0474b668fd43b294ece08957bf0648", + "e91b8ff7122d4842abb3aa0ff38e0352" + ] + }, + "atomic_rollback_and_5xx_retry": true, + "expired_key_can_be_reused": true, + "failovers": [ + { + "confirmed_writes": 194, + "successful_reads": 187, + "errors": 8, + "successful_rps": 25.32, + "first_success_after_action_seconds": 4.086, + "action": { + "finished": 3.064 + }, + "observations": [ + { + "at": 0.007, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.072, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.13, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.208, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.266, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.325, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.383, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.446, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.507, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.571, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.634, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.69, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.751, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.808, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.869, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.927, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 0.985, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 1.05, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 1.157, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 1.257, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 1.354, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 1.456, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 1.556, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 1.628, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 1.707, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 1.775, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 1.884, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 1.975, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 2.075, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 2.189, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 2.279, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "at": 2.349, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 2.424, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 2.628, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 2.731, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 2.834, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 2.927, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 3.022, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.086, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.149, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.208, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.267, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.324, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.382, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.445, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.511, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.573, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.634, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.701, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.755, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.819, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.886, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 4.953, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.018, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.087, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.155, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.239, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.313, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.388, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.454, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.541, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.616, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.683, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.758, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.833, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.9, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 5.968, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.036, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.105, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.168, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.238, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.317, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.388, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.451, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.517, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.577, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.646, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.709, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.765, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.821, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.886, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 6.951, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.02, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.086, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.151, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.221, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.291, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.366, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.436, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.512, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.587, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.654, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.718, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.787, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.859, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.925, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 7.992, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.063, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.136, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.211, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.276, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.348, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.43, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.497, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.561, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.628, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.71, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.772, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.844, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.909, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 8.977, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.042, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.106, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.166, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.23, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.299, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.361, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.428, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.499, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.567, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.639, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.713, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.786, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.862, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.928, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 9.99, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.055, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.117, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.187, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.254, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.321, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.388, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.462, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.53, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.6, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.676, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.745, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.818, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.884, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 10.95, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.007, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.069, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.138, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.2, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.259, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.318, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.383, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.449, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.507, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.574, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.644, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.718, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.787, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.854, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 11.931, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.018, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.087, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.158, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.228, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.297, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.368, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.438, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.506, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.571, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.641, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.709, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.781, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.86, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 12.928, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 13.001, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 13.068, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 13.137, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 13.225, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 13.424, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 13.52, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 13.63, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 13.726, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 13.822, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 13.93, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.024, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.092, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.15, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.217, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.285, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.352, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.423, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.492, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.56, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.627, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.694, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.768, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.836, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.902, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 14.968, + "instance": "ba0474b668fd43b294ece08957bf0648" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "c9c495521bf2", + "pid": 39030, + "started_at": "2026-11-15T18:03:15.404279669Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + } + }, + { + "confirmed_writes": 217, + "successful_reads": 211, + "errors": 7, + "successful_rps": 28.43, + "first_success_after_action_seconds": 2.72, + "action": { + "finished": 2.697 + }, + "observations": [ + { + "at": 0.013, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.08, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.148, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.214, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.278, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.352, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.417, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.475, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.536, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.589, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.647, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.71, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.774, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.872, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 0.983, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 1.072, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 1.175, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 1.275, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 1.376, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 1.44, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 1.545, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 1.647, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 1.744, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 1.848, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 1.947, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 2.01, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 2.063, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "at": 2.13, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.191, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.246, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.308, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.36, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.412, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.471, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.533, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.596, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.659, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.72, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.786, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.853, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.914, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.977, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.033, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.096, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.163, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.242, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.308, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.376, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.446, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.516, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.576, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.636, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.702, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.761, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.823, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.879, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.942, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.001, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.058, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.121, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.196, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.262, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.336, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.404, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.479, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.545, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.612, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.676, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.741, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.8, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.861, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.93, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.003, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.074, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.148, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.22, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.292, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.363, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.449, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.522, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.592, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.663, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.738, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.811, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.882, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.948, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.008, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.07, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.133, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.194, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.259, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.317, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.383, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.443, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.51, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.578, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.645, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.71, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.778, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.843, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.91, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.977, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.053, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.126, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.197, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.264, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.329, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.397, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.463, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.532, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.607, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.675, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.732, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.803, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.878, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.949, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.024, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.096, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.167, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.24, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.299, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.366, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.43, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.492, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.559, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.623, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.691, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.754, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.811, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.875, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.942, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.011, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.078, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.146, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.219, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.295, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.371, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.443, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.525, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.622, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.724, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.829, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.93, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.02, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.08, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.176, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.243, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.31, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.378, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.447, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.51, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.575, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.64, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.7, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.759, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.826, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.891, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.954, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.013, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.07, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.134, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.203, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.272, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.348, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.415, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.489, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.563, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.637, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.707, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.77, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.847, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.92, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.992, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.066, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.133, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.204, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.278, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.344, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.412, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.474, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.539, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.615, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.687, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.756, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.82, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.885, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.943, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.009, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.075, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.142, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.207, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.272, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.334, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.397, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.459, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.529, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.603, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.669, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.745, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.812, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.879, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.946, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.013, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.081, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.146, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.214, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.296, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.363, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.428, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.498, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.564, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.63, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.705, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.77, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.836, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.898, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.964, + "instance": "6a4f6dc699ed419dad620c988fe79946" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "2af84f3c87dd", + "pid": 80780, + "started_at": "2026-11-15T18:13:52.321424672Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "ba0474b668fd43b294ece08957bf0648" + } + } + ], + "rolling": { + "confirmed_writes": 609, + "successful_reads": 608, + "errors": 34, + "successful_rps": 26.97, + "first_success_after_action_seconds": 40.906, + "action": { + "finished": 40.838 + }, + "observations": [ + { + "at": 0.016, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.108, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.162, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.228, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.294, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.362, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.431, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.496, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.585, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.66, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.717, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.78, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.849, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.909, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 0.964, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.024, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.078, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.148, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.212, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.268, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.417, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.493, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.558, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.62, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.684, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.744, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.803, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.87, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 1.954, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.017, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.084, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.141, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.198, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.257, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.314, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.376, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.459, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.525, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.603, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.68, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.74, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.806, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.86, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.913, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 2.976, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.036, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.096, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.161, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.224, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.292, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.35, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.409, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.471, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.537, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.597, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.655, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.766, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.834, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.897, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 3.959, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.018, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.086, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.15, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.213, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.278, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.34, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.402, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.463, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.524, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.59, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.651, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.711, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.781, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.837, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.9, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 4.958, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.016, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.08, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.139, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.203, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.265, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.327, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.391, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.459, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.514, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.58, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.646, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.707, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.777, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.842, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.905, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 5.964, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.021, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.08, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.148, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.206, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.272, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.339, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.397, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.463, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.525, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.59, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.657, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.748, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.806, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.865, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.928, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 6.988, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.057, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.124, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.189, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.255, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.316, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.38, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.444, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.501, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.568, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.633, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.704, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.812, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 7.907, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.015, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.115, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.213, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.301, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.37, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.437, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.501, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.571, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.639, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.7, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.78, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.84, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.899, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 8.965, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.028, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.09, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.148, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.207, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.266, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.328, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.396, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.478, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.544, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.601, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.659, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.727, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.793, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.859, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.93, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 9.997, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.061, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.12, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.18, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.243, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.307, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.381, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.446, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.506, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.569, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.628, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.687, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.763, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.845, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.911, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 10.969, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.036, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.104, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.171, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.238, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.302, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.364, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.429, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.501, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.57, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.631, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.699, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.761, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.834, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.906, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 11.975, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.05, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.114, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.191, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.256, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.326, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.394, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.458, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.543, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.605, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.673, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.746, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.843, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 12.96, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.025, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.088, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.151, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.229, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.309, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.508, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.61, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.68, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.75, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.836, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 13.949, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.048, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.178, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 14.342, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 14.415, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.475, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 14.543, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 14.603, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.676, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 14.738, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 14.807, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 14.878, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 14.955, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 15.025, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 15.117, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 15.209, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 15.274, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 15.342, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 15.408, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 15.48, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 15.552, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 15.62, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 15.689, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 15.769, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 15.833, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 15.905, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 15.978, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 16.044, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 16.11, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 16.183, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 16.267, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 16.34, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 16.402, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 16.462, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 16.525, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 16.593, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 16.655, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 16.718, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 16.786, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 16.846, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 16.916, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 17.006, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 17.074, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 17.147, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 17.212, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 17.283, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 17.353, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "at": 17.427, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 17.497, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 17.565, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 17.64, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 17.716, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 17.813, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 17.888, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 17.964, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 18.027, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 18.114, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 18.175, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 18.25, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 19.426, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 19.502, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 19.581, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 19.653, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 19.727, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 19.801, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 19.878, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 19.943, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.007, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.074, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.134, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.208, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.284, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.355, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.415, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.477, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.543, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.609, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.674, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.736, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.8, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.86, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.923, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 20.988, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.046, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.113, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.182, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.251, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.318, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.39, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.458, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.53, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.603, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.668, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.735, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.798, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.86, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.928, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 21.994, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 22.056, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 22.117, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 22.186, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 22.259, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 22.332, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 22.398, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 22.46, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 22.527, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 22.59, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 22.65, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 22.727, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 22.794, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 22.871, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 22.944, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.013, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.081, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.161, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.228, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.292, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.359, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.426, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.496, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.557, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.627, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.695, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.761, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.831, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.898, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 23.964, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 24.029, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 24.095, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 24.164, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 24.235, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 24.328, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 24.426, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 24.53, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 24.628, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 24.728, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 24.826, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 24.927, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.013, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.084, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.15, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.217, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.286, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.346, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.407, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.471, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.535, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.599, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.664, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.727, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.789, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.854, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.921, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 25.983, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.058, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.123, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.192, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.259, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.325, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.393, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.458, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.525, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.59, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.651, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.715, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.776, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.837, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.903, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 26.97, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.052, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.129, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.194, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.275, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.339, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.412, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.497, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.561, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.625, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.684, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.744, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.8, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.856, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.915, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 27.983, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.05, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.115, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.172, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.235, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.296, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.357, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.425, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.484, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.549, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.625, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.694, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.759, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.826, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.888, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 28.956, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 29.037, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 29.118, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 29.214, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 29.326, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 29.421, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 29.509, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 29.621, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 29.717, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 29.812, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 29.905, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 29.964, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.03, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.091, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.16, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.224, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.286, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.348, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.414, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.474, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.542, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.599, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.657, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.72, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.79, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.851, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.918, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 30.991, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.055, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.12, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.188, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.252, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.32, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.382, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.446, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.514, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.577, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.644, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.707, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.773, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.84, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.905, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 31.977, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.042, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.115, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.18, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.241, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.314, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.374, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.443, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.508, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.578, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.636, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.702, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.761, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.825, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.883, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 32.95, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 33.013, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 33.074, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 33.137, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 33.197, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 33.261, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 33.334, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 33.417, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 33.475, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 33.556, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 33.627, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 33.69, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 33.752, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 33.812, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 33.87, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 33.938, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 34.005, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 34.073, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 34.13, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 34.192, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 34.254, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 34.312, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 34.375, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 34.445, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 34.511, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 34.574, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 34.643, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 34.705, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 34.775, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 34.837, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 34.906, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 34.97, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 35.048, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 35.124, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 35.192, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 35.253, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 35.316, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 35.387, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 35.521, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 35.584, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 35.694, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 35.758, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 35.826, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 35.893, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 35.964, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 36.038, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 36.107, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 36.174, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 36.248, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 36.315, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 36.388, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 36.459, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 36.53, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 36.602, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 36.666, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 36.729, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "at": 36.794, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "at": 36.863, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 36.934, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 37.001, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 37.083, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 37.153, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 37.224, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 37.293, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 37.368, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 37.458, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 38.603, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 38.68, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 38.755, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 38.823, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 38.894, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 38.959, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.025, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.096, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.165, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.231, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.291, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.353, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.416, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.503, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.576, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.643, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.708, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.775, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.841, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.91, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 39.979, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.052, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.118, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.186, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.252, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.319, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.391, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.463, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.538, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.612, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.686, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.762, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.831, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.906, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 40.978, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.052, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.121, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.201, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.272, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.349, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.413, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.476, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.565, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.634, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.695, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.758, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.819, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.883, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 41.945, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.009, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.078, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.145, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.218, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.284, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.421, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.517, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.578, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.643, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.708, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.774, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.838, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.909, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 42.977, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 43.045, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 43.118, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 43.196, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 43.306, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 43.405, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 43.515, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 43.706, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 43.813, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 43.923, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 44.015, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 44.102, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 44.163, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 44.253, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 44.324, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 44.404, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 44.483, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 44.671, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 44.752, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 44.826, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 44.91, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "at": 44.976, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "before": [ + { + "role": "api", + "container": "c9c495521bf2", + "pid": 536, + "started_at": "2026-11-15T18:23:01.344627468Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "6a4f6dc699ed419dad620c988fe79946" + }, + { + "role": "api", + "container": "2af84f3c87dd", + "pid": 1911, + "started_at": "2026-11-15T18:23:32.450473386Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "940942b16d664ce38af7c0db8fa979fa" + }, + { + "role": "cache", + "container": "e33ddf35f293", + "pid": 5356, + "started_at": "2026-11-15T17:39:31.281698344Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "lb", + "container": "6d7d02d323ed", + "pid": 39796, + "started_at": "2026-11-15T18:03:26.954206841Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8084" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "a5ffac5a4228", + "pid": 5418, + "started_at": "2026-11-15T17:39:31.614022427Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "c94b86137075", + "pid": 94299, + "started_at": "2026-11-15T18:20:05.941649137Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "s3", + "container": "acc08930aff2", + "pid": 82901, + "started_at": "2026-11-15T18:14:38.360606722Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + } + ], + "after": [ + { + "role": "api", + "container": "99db7624c4ec", + "pid": 2618, + "started_at": "2026-11-15T18:23:49.098497838Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "7675c0f93b74423d8c2206f5492f21f3" + }, + { + "role": "api", + "container": "b0a345f04e5b", + "pid": 3409, + "started_at": "2026-11-15T18:24:08.65538843Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "19362ec512e143f6b7a5aa8ddc8109c2" + }, + { + "role": "cache", + "container": "e33ddf35f293", + "pid": 5356, + "started_at": "2026-11-15T17:39:31.281698344Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "lb", + "container": "6d7d02d323ed", + "pid": 39796, + "started_at": "2026-11-15T18:03:26.954206841Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8084" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "a5ffac5a4228", + "pid": 5418, + "started_at": "2026-11-15T17:39:31.614022427Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "c94b86137075", + "pid": 94299, + "started_at": "2026-11-15T18:20:05.941649137Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "s3", + "container": "acc08930aff2", + "pid": 82901, + "started_at": "2026-11-15T18:14:38.360606722Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + } + ] + }, + "replay_after_replacement": true, + "public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 123, \"failed_attempts\": 3, \"first_recovery_seconds\": 2.635}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.", + "after_persistence": [ + { + "role": "api", + "container": "99db7624c4ec", + "pid": 7567, + "started_at": "2026-11-15T18:25:23.088247673Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "1bc05e903ca54f3498aca9b1cec61bdb" + }, + { + "role": "api", + "container": "b0a345f04e5b", + "pid": 7483, + "started_at": "2026-11-15T18:25:22.717902089Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "5383b6087a8c478bb7ed3ec4371c194c" + }, + { + "role": "cache", + "container": "e33ddf35f293", + "pid": 7268, + "started_at": "2026-11-15T18:25:21.717046214Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "lb", + "container": "6d7d02d323ed", + "pid": 7163, + "started_at": "2026-11-15T18:25:21.610361422Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8084" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "a5ffac5a4228", + "pid": 7246, + "started_at": "2026-11-15T18:25:21.70517763Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "c94b86137075", + "pid": 94299, + "started_at": "2026-11-15T18:20:05.941649137Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "s3", + "container": "acc08930aff2", + "pid": 7196, + "started_at": "2026-11-15T18:25:21.649267714Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + } + ] +} diff --git a/evidence/http-first.json b/evidence/http-first.json new file mode 100644 index 00000000..851b23db --- /dev/null +++ b/evidence/http-first.json @@ -0,0 +1,17 @@ +{ + "date": "2026-11-15T18:04:52.576254+00:00", + "source_commit": "f5a7a74f679d936d851fa0e58f6cfb59dcd62c50", + "result": "PASS", + "smoke": { + "command": "make test PYTHON=.venv/bin/python BASE_URL=http://127.0.0.1:8084", + "seconds": 8.702, + "exit_code": 0, + "output": ".venv/bin/python tests/smoke.py --base-url \"http://127.0.0.1:8084\" \nPASS: lab4 public API scenario; infrastructure and grading evidence remain separate.\n" + }, + "http": { + "command": ".venv/bin/pytest student-tests -q", + "seconds": 362.959, + "exit_code": 0, + "output": "........................................................................ [ 66%]\n.................................... [100%]\n108 passed in 362.40s (0:06:02)\n" + } +} diff --git a/evidence/minio-source-verification.json b/evidence/minio-source-verification.json index f3d8eb3e..1da57808 100644 --- a/evidence/minio-source-verification.json +++ b/evidence/minio-source-verification.json @@ -8,5 +8,85 @@ "module_checksum": "h1:6TdolSCLSs2nwm8i0PpWDqf9iX2Ty9WQK8wmr7dCnUM=", "go_mod_checksum": "h1:yCWDkwWO9IWpGsT4mreDDN/B/QVmK2zC666uInRAcqE=", "validation": "standard go mod download with default Go checksum database after ranged HTTPS download", - "compiler_build_status": "in progress; source validation only" + "compiler_build_status": "PASS", + "runtime_version": "minio version RELEASE.2025-10-15T17-29-55Z (commit-id=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a)\nRuntime: go1.24.8 linux/arm64\nLicense: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html\nCopyright: 2015-0000 MinIO, Inc.", + "runtime_snapshot": [ + { + "role": "api", + "container": "c9c495521bf2", + "pid": 39030, + "started_at": "2026-11-15T18:03:15.404279669Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "role": "api", + "container": "2af84f3c87dd", + "pid": 39163, + "started_at": "2026-11-15T18:03:15.735686586Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "17cd27ba20064c5eb9580379e963f419" + }, + { + "role": "cache", + "container": "e33ddf35f293", + "pid": 5356, + "started_at": "2026-11-15T17:39:31.281698344Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "lb", + "container": "6d7d02d323ed", + "pid": 39796, + "started_at": "2026-11-15T18:03:26.954206841Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8084" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "a5ffac5a4228", + "pid": 5418, + "started_at": "2026-11-15T17:39:31.614022427Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "c94b86137075", + "pid": 39092, + "started_at": "2026-11-15T18:03:15.578851294Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "s3", + "container": "acc08930aff2", + "pid": 38412, + "started_at": "2026-11-15T18:03:03.039434803Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + } + ], + "runtime_image_revision": { + "api": "f5a7a74f679d936d851fa0e58f6cfb59dcd62c50", + "s3": "9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a", + "bootstrap": "7394ce0dd2a80935aded936b09fa12cbb3cb8096" + } } diff --git a/evidence/storage-first.json b/evidence/storage-first.json new file mode 100644 index 00000000..91f2434b --- /dev/null +++ b/evidence/storage-first.json @@ -0,0 +1,502 @@ +{ + "source_commit": "f5a7a74f679d936d851fa0e58f6cfb59dcd62c50", + "date": "2026-11-15T18:13:30.360880+00:00", + "complete": true, + "runtime": [ + { + "role": "api", + "container": "c9c495521bf2", + "pid": 39030, + "started_at": "2026-11-15T18:03:15.404279669Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "role": "api", + "container": "2af84f3c87dd", + "pid": 39163, + "started_at": "2026-11-15T18:03:15.735686586Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "17cd27ba20064c5eb9580379e963f419" + }, + { + "role": "cache", + "container": "e33ddf35f293", + "pid": 5356, + "started_at": "2026-11-15T17:39:31.281698344Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "lb", + "container": "6d7d02d323ed", + "pid": 39796, + "started_at": "2026-11-15T18:03:26.954206841Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8084" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "a5ffac5a4228", + "pid": 5418, + "started_at": "2026-11-15T17:39:31.614022427Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "c94b86137075", + "pid": 39092, + "started_at": "2026-11-15T18:03:15.578851294Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "s3", + "container": "acc08930aff2", + "pid": 38412, + "started_at": "2026-11-15T18:03:03.039434803Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + } + ], + "upload_replica_stopped_downloaded_elsewhere": { + "stopped_container": "2af84f3c87dd", + "uploader": "17cd27ba20064c5eb9580379e963f419", + "downloader": "e91b8ff7122d4842abb3aa0ff38e0352", + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + }, + "private_bucket_and_iam": { + "denied_head_is_not_missing": 503, + "anonymous_get": 403, + "list_other_bucket": "AccessDenied", + "create_bucket": "AccessDenied", + "write_outside_prefix": "AccessDenied", + "console_internal_network": "unreachable" + }, + "bootstrap_repeat_preserved_original": true, + "aborted_upload_has_no_intent_or_bytes": true, + "pg_intent_failure_has_no_bytes": true, + "s3_unavailable": { + "status": 503, + "error": "unavailable" + }, + "concurrent_uploads": { + "concurrency": 10, + "requests": 10, + "successes": 10, + "errors": 0, + "bytes_per_image": 6017095, + "pixels": 2890000, + "duration_seconds": 5.149, + "successful_rps": 1.942, + "latency_scope": "upload only; duration includes verified downloads and memory sampler", + "p50_ms": 2162.965, + "p95_ms": 4465.512, + "p99_ms": 4465.512, + "cpu_before": { + "c9c495521bf2e0eb21d94eebc8ac06ea3735ee282401f8bbf4918203b05e69ea": "usage_usec 148242470\nuser_usec 139413925\nsystem_usec 8828544\nnice_usec 0\nnr_periods 5268\nnr_throttled 2720\nthrottled_usec 566509749\nnr_bursts 0\nburst_usec 0", + "2af84f3c87dd51105775135adaa2864b852c68c3b3d929f566d714ee4bbf13a2": "usage_usec 7938759\nuser_usec 7311622\nsystem_usec 627137\nnice_usec 0\nnr_periods 338\nnr_throttled 144\nthrottled_usec 17342812\nnr_bursts 0\nburst_usec 0" + }, + "cpu_after": { + "c9c495521bf2e0eb21d94eebc8ac06ea3735ee282401f8bbf4918203b05e69ea": "usage_usec 150694911\nuser_usec 141253264\nsystem_usec 9441647\nnice_usec 0\nnr_periods 5320\nnr_throttled 2767\nthrottled_usec 573793811\nnr_bursts 0\nburst_usec 0", + "2af84f3c87dd51105775135adaa2864b852c68c3b3d929f566d714ee4bbf13a2": "usage_usec 9567439\nuser_usec 8568377\nsystem_usec 999061\nnice_usec 0\nnr_periods 386\nnr_throttled 171\nthrottled_usec 20452342\nnr_bursts 0\nburst_usec 0" + }, + "memory_samples": [ + { + "at": 0.0, + "container": "c9c495521bf2", + "memory.current": 128004096, + "memory.peak": 215093248 + }, + { + "at": 0.931, + "container": "2af84f3c87dd", + "memory.current": 128983040, + "memory.peak": 152731648 + }, + { + "at": 2.318, + "container": "c9c495521bf2", + "memory.current": 163500032, + "memory.peak": 215093248 + }, + { + "at": 3.846, + "container": "2af84f3c87dd", + "memory.current": 126009344, + "memory.peak": 152731648 + }, + { + "at": 4.515, + "container": "c9c495521bf2", + "memory.current": 153980928, + "memory.peak": 215093248 + }, + { + "at": 4.862, + "container": "2af84f3c87dd", + "memory.current": 125583360, + "memory.peak": 152731648 + } + ], + "results": [ + { + "id": "7df54468-dffb-48e1-b435-bb64e73b087c", + "latency_ms": 774.2156249878462, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "id": "e1ba3763-89f2-497f-a152-6008f4737e08", + "latency_ms": 1630.1402079989202, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "id": "69c854d8-ca53-46aa-9dbd-646b68bc42be", + "latency_ms": 3091.3296249927953, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "id": "03328959-bf30-4e57-a233-4084e599bbfc", + "latency_ms": 2098.3562500041444, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "id": "483eff54-3137-4764-bc89-3feccf7d00e6", + "latency_ms": 4303.714374982519, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "id": "f4eb43f0-65bf-462c-bfd2-9c8d88ff1d34", + "latency_ms": 1105.6364579999354, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "id": "90a80cf9-d955-4bcc-9f16-01587a3f2f66", + "latency_ms": 3825.5119579844177, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "id": "3e2a2c9c-4204-4883-b18a-5b1deeca70c3", + "latency_ms": 2162.9652910050936, + "instance": "ba0474b668fd43b294ece08957bf0648" + }, + { + "id": "ea4b8e91-80fe-4741-8d0b-c77d2da6656f", + "latency_ms": 4465.512333001243, + "instance": "e91b8ff7122d4842abb3aa0ff38e0352" + }, + { + "id": "031a9c2b-4c93-4dd4-935b-8104080a0a86", + "latency_ms": 1169.7680839861277, + "instance": "ba0474b668fd43b294ece08957bf0648" + } + ], + "window_max_memory_current": 163500032, + "lifetime_memory_peak": 215093248 + }, + "reconciliation": { + "dry_run": { + "apply": false, + "grace_seconds": 300, + "objects_scanned": 20, + "rows_scanned": 20, + "head_requests": 8, + "bytes_read": 111817, + "fresh_protected": 13, + "ready_protected": 5, + "linked_protected": 0, + "actions": { + "delete_orphan": 1, + "recover_pending": 1, + "fail_pending_missing": 1, + "delete_failed_object": 1, + "mark_missing": 1 + }, + "applied": {}, + "details": [ + { + "action": "delete_orphan", + "identity": "objects/verification-f49e0d70d0f54f62bce62b4a3dccdbdc/orphan" + }, + { + "action": "recover_pending", + "identity": "06e24ef9-2a84-43d1-bc08-facf33155e9f" + }, + { + "action": "fail_pending_missing", + "identity": "44b0d154-ed76-4f79-a356-7feb85a3b14c" + }, + { + "action": "delete_failed_object", + "identity": "54e914d0-5a4c-4dc9-982c-ef887467700c" + }, + { + "action": "mark_missing", + "identity": "fa66fcd8-ca7e-41e4-8f25-25ff119c38e9" + } + ], + "duration_seconds": 0.0604, + "complete": true + }, + "apply": { + "apply": true, + "grace_seconds": 300, + "objects_scanned": 20, + "rows_scanned": 20, + "head_requests": 10, + "bytes_read": 111817, + "fresh_protected": 13, + "ready_protected": 5, + "linked_protected": 0, + "actions": { + "delete_orphan": 1, + "recover_pending": 1, + "fail_pending_missing": 1, + "delete_failed_object": 1, + "mark_missing": 1 + }, + "applied": { + "delete_orphan": 1, + "recover_pending": 1, + "fail_pending_missing": 1, + "delete_failed_object": 1, + "mark_missing": 1 + }, + "details": [ + { + "action": "delete_orphan", + "identity": "objects/verification-f49e0d70d0f54f62bce62b4a3dccdbdc/orphan" + }, + { + "action": "recover_pending", + "identity": "06e24ef9-2a84-43d1-bc08-facf33155e9f" + }, + { + "action": "fail_pending_missing", + "identity": "44b0d154-ed76-4f79-a356-7feb85a3b14c" + }, + { + "action": "delete_failed_object", + "identity": "54e914d0-5a4c-4dc9-982c-ef887467700c" + }, + { + "action": "mark_missing", + "identity": "fa66fcd8-ca7e-41e4-8f25-25ff119c38e9" + } + ], + "duration_seconds": 0.106004, + "complete": true + }, + "repeat": { + "apply": true, + "grace_seconds": 300, + "objects_scanned": 18, + "rows_scanned": 20, + "head_requests": 6, + "bytes_read": 0, + "fresh_protected": 13, + "ready_protected": 6, + "linked_protected": 1, + "actions": {}, + "applied": {}, + "details": [], + "duration_seconds": 0.102749, + "complete": true + }, + "s3_grace_wait_seconds": 331.301, + "fresh_and_ready_unlinked_protected": true, + "old_failed_unlinked_deleted_and_fresh_failed_protected": true, + "missing_used_id_and_link_preserved": true, + "dry_run_unchanged": true + }, + "cost_research": { + "cells": [ + { + "extra_objects": 0, + "repetition": 1, + "apply": false, + "grace_seconds": 300, + "objects_scanned": 18, + "rows_scanned": 20, + "head_requests": 6, + "bytes_read": 0, + "fresh_protected": 13, + "ready_protected": 6, + "linked_protected": 1, + "actions": {}, + "applied": {}, + "details": [], + "duration_seconds": 0.183485, + "complete": true + }, + { + "extra_objects": 0, + "repetition": 2, + "apply": false, + "grace_seconds": 300, + "objects_scanned": 18, + "rows_scanned": 20, + "head_requests": 6, + "bytes_read": 0, + "fresh_protected": 13, + "ready_protected": 6, + "linked_protected": 1, + "actions": {}, + "applied": {}, + "details": [], + "duration_seconds": 0.043382, + "complete": true + }, + { + "extra_objects": 0, + "repetition": 3, + "apply": false, + "grace_seconds": 300, + "objects_scanned": 18, + "rows_scanned": 20, + "head_requests": 6, + "bytes_read": 0, + "fresh_protected": 13, + "ready_protected": 6, + "linked_protected": 1, + "actions": {}, + "applied": {}, + "details": [], + "duration_seconds": 0.046774, + "complete": true + }, + { + "extra_objects": 100, + "repetition": 1, + "apply": false, + "grace_seconds": 300, + "objects_scanned": 118, + "rows_scanned": 20, + "head_requests": 6, + "bytes_read": 0, + "fresh_protected": 112, + "ready_protected": 6, + "linked_protected": 1, + "actions": {}, + "applied": {}, + "details": [], + "duration_seconds": 0.078726, + "complete": true + }, + { + "extra_objects": 100, + "repetition": 2, + "apply": false, + "grace_seconds": 300, + "objects_scanned": 118, + "rows_scanned": 20, + "head_requests": 6, + "bytes_read": 0, + "fresh_protected": 112, + "ready_protected": 6, + "linked_protected": 1, + "actions": {}, + "applied": {}, + "details": [], + "duration_seconds": 0.088476, + "complete": true + }, + { + "extra_objects": 100, + "repetition": 3, + "apply": false, + "grace_seconds": 300, + "objects_scanned": 118, + "rows_scanned": 20, + "head_requests": 6, + "bytes_read": 0, + "fresh_protected": 112, + "ready_protected": 6, + "linked_protected": 1, + "actions": {}, + "applied": {}, + "details": [], + "duration_seconds": 0.206964, + "complete": true + }, + { + "extra_objects": 1100, + "repetition": 1, + "apply": false, + "grace_seconds": 300, + "objects_scanned": 1118, + "rows_scanned": 20, + "head_requests": 6, + "bytes_read": 0, + "fresh_protected": 1112, + "ready_protected": 6, + "linked_protected": 1, + "actions": {}, + "applied": {}, + "details": [], + "duration_seconds": 0.197227, + "complete": true + }, + { + "extra_objects": 1100, + "repetition": 2, + "apply": false, + "grace_seconds": 300, + "objects_scanned": 1118, + "rows_scanned": 20, + "head_requests": 10, + "bytes_read": 0, + "fresh_protected": 1108, + "ready_protected": 10, + "linked_protected": 1, + "actions": {}, + "applied": {}, + "details": [], + "duration_seconds": 0.145977, + "complete": true + }, + { + "extra_objects": 1100, + "repetition": 3, + "apply": false, + "grace_seconds": 300, + "objects_scanned": 1118, + "rows_scanned": 20, + "head_requests": 16, + "bytes_read": 0, + "fresh_protected": 1102, + "ready_protected": 16, + "linked_protected": 1, + "actions": {}, + "applied": {}, + "details": [], + "duration_seconds": 0.199997, + "complete": true + } + ], + "repeats": 3, + "method": "same running stand; 0/100/1100 fresh one-byte objects; dry-run; serial ascending dataset; no warmup", + "median_seconds": { + "0": 0.046774, + "100": 0.088476, + "1100": 0.197227 + } + } +} diff --git a/evidence/storage-research.png b/evidence/storage-research.png new file mode 100644 index 00000000..0008d204 Binary files /dev/null and b/evidence/storage-research.png differ diff --git a/research/cost.py b/research/cost.py new file mode 100644 index 00000000..b331a6ca --- /dev/null +++ b/research/cost.py @@ -0,0 +1,159 @@ +"""Controlled inventory experiment on the teaching stand, with own fixtures.""" + +import argparse +import json +import statistics +from datetime import UTC, datetime +from pathlib import Path +from uuid import uuid4 + +import httpx + +from scripts.stand import BASE, actor, auth, command, compose, ready, sql +from scripts.verify_storage import PNG, PREFIX, inside + + +def probe(): + # Run the actual code with the reconciler service's .25 CPU /256 MiB. + code = ( + PREFIX + + """ +import pathlib,resource +started=time.process_time() +result=reconcile(d,s,apply=False,grace=3600) +result['cpu_seconds']=time.process_time()-started +result['rss_peak_bytes']=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024 +result['cgroup_memory_peak_bytes']=int(pathlib.Path('/sys/fs/cgroup/memory.peak').read_text()) +print(json.dumps(result)); s.close(); d.close() +""" + ) + return json.loads( + command( + [ + "docker", + "compose", + "run", + "--rm", + "--no-deps", + "-T", + "reconciler", + "python", + "-", + ], + input=code, + ) + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="evidence/cost-repeat.json") + args = parser.parse_args() + output = Path(args.output) + assert not output.exists() + output.parent.mkdir(parents=True, exist_ok=True) + prefix = "objects/cost-" + uuid4().hex + "/" + result = { + "date": datetime.now(UTC).isoformat(), + "source_commit": command(["git", "rev-parse", "HEAD"]), + "complete": False, + "cells": [], + "warmups": [], + "method": "0/100/1100 extra fresh one-byte objects; one warmup then 3 repeats per dataset; real reconciler service .25 CPU/256MiB; grace3600; 8 PNG fixtures with deliberately aged PG metadata (2days); other runtime objects remain real", + "fixture_rows": 8, + "grace_seconds": 3600, + } + chat = None + ready() + compose("stop", "reconciler") + try: + with httpx.Client( + base_url=BASE, trust_env=False, timeout=20 + ) as client: + user, _, session = actor(client) + r = client.post( + "/api/v1/chats", + headers=auth(session), + json={ + "title": "Controlled inventory", + "member_ids": [user["id"]], + }, + ) + assert r.status_code == 201 + chat = r.json()["id"] + seed = f""" +from uuid import uuid4 +data=bytes.fromhex({PNG.hex()!r}) +for i in range(8): + identity=uuid4(); key={prefix!r}+'ready-'+str(i) + s.put(key=key,stream=io.BytesIO(data),size=len(data),content_type='image/png',sha256=hashlib.sha256(data).hexdigest()) + with d.connection() as c: + c.execute("INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,uploader_instance,expires_at,created_at) VALUES(%s,%s,%s,'fixture.png','image/png',%s,%s,%s,'ready','cost-fixture',clock_timestamp()-interval '1 day',clock_timestamp()-interval '2 days')",(identity,{chat!r},{user["id"]!r},len(data),hashlib.sha256(data).hexdigest(),key)) +print('true') +""" + inside(seed) + for count in [0, 100, 1100]: + if count: + start = 0 if count == 100 else 100 + inside( + f"\nfor i in range({start},{count}):\n s.put(key={prefix!r}+'extra-'+str(i),stream=io.BytesIO(b'x'),size=1,content_type='application/octet-stream',sha256=hashlib.sha256(b'x').hexdigest())\nprint('true')" + ) + result["warmups"].append({"extra_objects": count, **probe()}) + for repeat in range(3): + cell = { + "extra_objects": count, + "repetition": repeat + 1, + **probe(), + } + result["cells"].append(cell) + assert cell["complete"] and not cell["applied"] + assert cell["ready_protected"] >= 8 + controls = { + (v["rows_scanned"], v["head_requests"], v["bytes_read"]) + for v in result["cells"] + } + assert len(controls) == 1, ( + "Age boundary changed work; choose a fresh stand" + ) + for count in [0, 100, 1100]: + assert ( + len( + { + v["objects_scanned"] + for v in result["cells"] + if v["extra_objects"] == count + } + ) + == 1 + ) + result["stable_rows_head_bytes"] = list(next(iter(controls))) + result["median_seconds"] = { + str(n): statistics.median( + v["duration_seconds"] + for v in result["cells"] + if v["extra_objects"] == n + ) + for n in [0, 100, 1100] + } + result["complete"] = True + finally: + if chat: + sql( + "DELETE FROM attachments WHERE chat_id='" + + chat + + "' AND uploader_instance='cost-fixture'" + ) + inside( + f"\nfor o in s.objects():\n if o.object_name.startswith({prefix!r}): s.delete(key=o.object_name)\nprint('true')" + ) + compose("up", "-d", "--no-build", "--no-recreate", "reconciler") + output.write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n" + ) + print( + "PASS: controlled inventory cost, 9 measurements and constant metadata/HEAD work" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_storage.py b/scripts/verify_storage.py index 81c2e92d..e3944c19 100644 --- a/scripts/verify_storage.py +++ b/scripts/verify_storage.py @@ -386,7 +386,14 @@ except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':j inside(f"s.delete(key={key(used['id'])!r}); print('true')") age(used["id"]) age(saved["id"]) + # Fresh cgroups make memory.peak specific to this experiment. + compose("restart", "api") + cluster.wait_count(2) + ready() result["concurrent_uploads"] = concurrency(client, session, chat) + result["concurrent_uploads"][ + "api_restarted_before_memory_measurement" + ] = True while time.monotonic() - grace_started < 302: print( "Waiting for genuine S3 LastModified grace: " @@ -464,7 +471,7 @@ except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':j f"print(json.dumps(s.head(key={key(failed_old['id'])!r}) is None and s.head(key={key(failed_fresh['id'])!r}) is not None))" ) repeat = run_reconcile(True) - assert not repeat["applied"].get("delete_orphan") + assert not repeat["applied"] assert repeat["linked_protected"] >= 1 result["reconciliation"] = { "dry_run": dry,